From 680862e96a19ca6d067a28793cff2b5141785b1b Mon Sep 17 00:00:00 2001 From: Zefa Chen Date: Tue, 10 Jun 2025 18:11:59 +0800 Subject: [PATCH 01/49] media: rockchip: isp support attach thunderboot buffer when iommu enable Change-Id: I2ddf053c1f4780687dfb322b5ed4e27df03f7c7d Signed-off-by: Zefa Chen --- drivers/media/platform/rockchip/isp/rkisp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/platform/rockchip/isp/rkisp.c b/drivers/media/platform/rockchip/isp/rkisp.c index 05cce6ab4776..b51d6fc3ca95 100644 --- a/drivers/media/platform/rockchip/isp/rkisp.c +++ b/drivers/media/platform/rockchip/isp/rkisp.c @@ -3477,7 +3477,7 @@ static int rkisp_rx_buf_pool_init(struct rkisp_device *dev, "%s type:0x%x first:%d dbufs[%d]:%p", __func__, dbufs->type, dbufs->is_first, i, dbufs); - if (dbufs->is_resmem) { + if (dbufs->is_resmem && !dev->hw_dev->is_mmu) { dma = dbufs->dma; goto end; } From cceb0646babe508eb8b2c232a6e87b63a7bcc15d Mon Sep 17 00:00:00 2001 From: Wei Dun Date: Wed, 11 Jun 2025 17:36:42 +0800 Subject: [PATCH 02/49] media: rockchip: vpss: update vpss stream stop process Change-Id: I5cd198d8b741f4227099e6fc29c0601b50c454b3 Signed-off-by: Wei Dun --- drivers/media/platform/rockchip/vpss/stream_v20.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/rockchip/vpss/stream_v20.c b/drivers/media/platform/rockchip/vpss/stream_v20.c index aa01566bcc93..fb2defee33fc 100644 --- a/drivers/media/platform/rockchip/vpss/stream_v20.c +++ b/drivers/media/platform/rockchip/vpss/stream_v20.c @@ -1947,12 +1947,10 @@ static void rkvpss_stream_stop(struct rkvpss_stream *stream) int ret; stream->stopping = true; - if (atomic_read(&dev->pipe_stream_cnt) > 0) { - ret = wait_event_timeout(stream->done, !stream->streaming, - msecs_to_jiffies(300)); - if (!ret) - v4l2_warn(&dev->v4l2_dev, "%s id:%d timeout\n", __func__, stream->id); - } + ret = wait_event_timeout(stream->done, !stream->streaming, + msecs_to_jiffies(300)); + if (!ret) + v4l2_warn(&dev->v4l2_dev, "%s id:%d timeout\n", __func__, stream->id); stream->stopping = false; stream->streaming = false; if (stream->ops->disable_mi) From d028e66817bfd10d809a6d2b91ed3100d336992c Mon Sep 17 00:00:00 2001 From: Shawn Lin Date: Thu, 12 Jun 2025 09:41:46 +0800 Subject: [PATCH 03/49] PCI: rockchip: dw: Fix rk_pcie_remove() Fixes: fbdf4ef963fc ("PCI: rockchip: dw: Add remove() support") Signed-off-by: Shawn Lin Change-Id: I9f630c85a8022c02232f5898f35556f3eab20cf1 --- drivers/pci/controller/dwc/pcie-dw-rockchip.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c index a08fab7c9790..f9c84def2614 100644 --- a/drivers/pci/controller/dwc/pcie-dw-rockchip.c +++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c @@ -1789,7 +1789,8 @@ static int rk_pcie_remove(struct platform_device *pdev) * Timeout should not happen as it's longer than regular probe actually. * But probe maybe fail, so need to double check bridge bus. */ - if (!rk_pcie || !rk_pcie->finish_probe || !rk_pcie->pci->pp.bridge->bus) { + if (!rk_pcie || !rk_pcie->pci || !rk_pcie->pci->pp.bridge || + !rk_pcie->pci->pp.bridge->bus) { dev_dbg(dev, "%s return early due to failure in threaded init\n", __func__); return 0; } From 652ca6831424773d4f971c32a874137a9a2e854c Mon Sep 17 00:00:00 2001 From: Weixin Zhou Date: Fri, 6 Jun 2025 11:04:17 +0800 Subject: [PATCH 04/49] MALI: bifrost: add gpu mem sysfs entry ls -l sys/class/misc/mali0/device/kprcs/(pid_id)/ -r--r--r-- 1 root root 4096 2025-06-12 21:59 private_gpu_mem -r--r--r-- 1 root root 4096 2025-06-12 21:59 total_gpu_mem ls -l sys/class/misc/mali0/device/ -r--r--r-- 1 root root 4096 2025-06-12 22:35 private_gpu_mem -r--r--r-- 1 root root 4096 2025-06-12 22:35 total_gpu_mem Signed-off-by: Weixin Zhou Change-Id: I1e4d8687baeaa93b160af430f227d1bd243cb381 --- .../gpu/arm/bifrost/mali_kbase_core_linux.c | 133 +++++++++++++++++- drivers/gpu/arm/bifrost/mali_kbase_defs.h | 2 + 2 files changed, 133 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c index 9cd02fec4a10..0f41b0411faa 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c @@ -360,6 +360,8 @@ static void kbase_file_delete(struct kbase_file *const kfile) #endif kbase_context_debugfs_term(kctx); + kobject_put(&kctx->kobj); + kbase_destroy_context(kctx); dev_dbg(kbdev->dev, "deleted base context\n"); @@ -624,14 +626,74 @@ static const struct file_operations kbase_force_same_va_fops = { }; #endif /* CONFIG_DEBUG_FS */ +static ssize_t kbase_kctx_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct kobj_attribute *kattr = container_of(attr, struct kobj_attribute, attr); + + if (kattr->show) + return kattr->show(kobj, kattr, buf); + + return -EIO; +} + +static ssize_t kbase_total_gpu_mem_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct kbase_context *kctx = container_of(kobj, struct kbase_context, kobj); + + return scnprintf(buf, PAGE_SIZE, "%zu\n", kctx->kprcs->total_gpu_pages << PAGE_SHIFT); +} + +static ssize_t kbase_private_gpu_mem_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct kbase_context *kctx = container_of(kobj, struct kbase_context, kobj); + + return scnprintf(buf, PAGE_SIZE, "%d\n", atomic_read(&kctx->used_pages) << PAGE_SHIFT); +} + +static struct kobj_attribute kbase_total_gpu_mem_attr = { + .attr = { + .name = "total_gpu_mem", + .mode = 0444, + }, + .show = kbase_total_gpu_mem_show, + .store = NULL, +}; + +static struct kobj_attribute kbase_private_gpu_mem_attr = { + .attr = { + .name = "private_gpu_mem", + .mode = 0444, + }, + .show = kbase_private_gpu_mem_show, + .store = NULL, +}; + +static struct attribute *kbase_kctx_attrs[] = { + &kbase_total_gpu_mem_attr.attr, + &kbase_private_gpu_mem_attr.attr, + NULL, +}; + +static const struct attribute_group kbase_kctx_attr_group = { + .attrs = kbase_kctx_attrs, +}; + +static const struct sysfs_ops kbase_kctx_sysfs_ops = { + .show = kbase_kctx_attr_show, +}; + +static const struct kobj_type kbase_kctx_ktype = { + .sysfs_ops = &kbase_kctx_sysfs_ops, + .default_groups = (const struct attribute_group *[]) { &kbase_kctx_attr_group, NULL }, +}; + static int kbase_file_create_kctx(struct kbase_file *const kfile, base_context_create_flags const flags) { struct kbase_device *kbdev = NULL; struct kbase_context *kctx = NULL; -#if IS_ENABLED(CONFIG_DEBUG_FS) char kctx_name[64]; -#endif + int ret = 0; if (WARN_ON(!kfile)) return -EINVAL; @@ -676,6 +738,12 @@ static int kbase_file_create_kctx(struct kbase_file *const kfile, } #endif /* CONFIG_DEBUG_FS */ + ret = kobject_init_and_add(&kctx->kobj, &kbase_kctx_ktype, kbdev->kprcs_kobj, kctx_name); + if (ret) { + dev_err(kbdev->dev, "Failed to create kctx kobject"); + kobject_put(&kctx->kobj); + } + dev_dbg(kbdev->dev, "created base context\n"); kfile->kctx = kctx; @@ -3182,6 +3250,56 @@ static ssize_t gpuinfo_show(struct device *dev, struct device_attribute *attr, c } static DEVICE_ATTR_RO(gpuinfo); +/** + * gpumem_private_show - Show callback for the gpumem_private sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU memory information. + * + * This function is called to get the current number of pages used by the GPU. + * The returned value is in bytes. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t private_gpu_mem_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct kbase_device *kbdev; + + CSTD_UNUSED(attr); + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + return scnprintf(buf, PAGE_SIZE, "%llu\n", (u64)atomic_read(&(kbdev->memdev.used_pages)) << PAGE_SHIFT); +} +static DEVICE_ATTR_RO(private_gpu_mem); + +/** + * total_gpu_mem_show - Show callback for the total_gpu_mem sysfs entry. + * @dev: The device this sysfs file is for. + * @attr: The attributes of the sysfs file. + * @buf: The output buffer to receive the GPU memory information. + * + * This function is called to get the total GPU memory including dmabuf memory. + * The returned value is in bytes. + * + * Return: The number of bytes output to @buf. + */ +static ssize_t total_gpu_mem_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct kbase_device *kbdev; + + CSTD_UNUSED(attr); + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + return scnprintf(buf, PAGE_SIZE, "%zu\n", kbdev->total_gpu_pages << PAGE_SHIFT); +} +static DEVICE_ATTR_RO(total_gpu_mem); + /** * dvfs_period_store - Store callback for the dvfs_period sysfs file. * @dev: The device with sysfs file is for @@ -5556,6 +5674,8 @@ static struct attribute *kbase_attrs[] = { &dev_attr_soft_job_timeout.attr, #endif /* !MALI_USE_CSF */ &dev_attr_gpuinfo.attr, + &dev_attr_total_gpu_mem.attr, + &dev_attr_private_gpu_mem.attr, &dev_attr_dvfs_period.attr, &dev_attr_pm_poweroff.attr, &dev_attr_reset_timeout.attr, @@ -5631,6 +5751,14 @@ int kbase_sysfs_init(struct kbase_device *kbdev) sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); } + kbdev->kprcs_kobj = kobject_create_and_add("kprcs", &kbdev->dev->kobj); + if (!kbdev->kprcs_kobj) { + dev_err(kbdev->dev, "Creation of kprcs sysfs group failed"); + sysfs_remove_group(&kbdev->dev->kobj, &kbase_mempool_attr_group); + sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); + sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); + } + return err; } @@ -5639,6 +5767,7 @@ void kbase_sysfs_term(struct kbase_device *kbdev) sysfs_remove_group(&kbdev->dev->kobj, &kbase_mempool_attr_group); sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); + kobject_put(kbdev->kprcs_kobj); put_device(kbdev->dev); } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_defs.h index b97df15f7a17..b7b19b802eae 100755 --- a/drivers/gpu/arm/bifrost/mali_kbase_defs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_defs.h @@ -1109,6 +1109,7 @@ struct kbase_device { unsigned int nr_regulators; #endif /* CONFIG_REGULATOR */ char devname[DEVNAME_SIZE]; + struct kobject *kprcs_kobj; u32 id; #if !IS_ENABLED(CONFIG_MALI_REAL_HW) @@ -1901,6 +1902,7 @@ struct kbase_sub_alloc { * is made on the device file. */ struct kbase_context { + struct kobject kobj; struct file *filp; struct kbase_device *kbdev; struct list_head kctx_list_link; From a37464fa3d676c5d14da816f0e158c8987f4fd89 Mon Sep 17 00:00:00 2001 From: XiaoDong Huang Date: Fri, 13 Jun 2025 14:58:37 +0800 Subject: [PATCH 05/49] firmware: rockchip_sip: add some subfunctions for gpio_config Signed-off-by: XiaoDong Huang Change-Id: I612aaa193c4bef98a43276e9ae85c13a702995d2 --- include/linux/rockchip/rockchip_sip.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/linux/rockchip/rockchip_sip.h b/include/linux/rockchip/rockchip_sip.h index 0ebcb2fae62d..5e42cf249e71 100644 --- a/include/linux/rockchip/rockchip_sip.h +++ b/include/linux/rockchip/rockchip_sip.h @@ -256,6 +256,9 @@ enum { GPIO_GET_GROUP_INFO = 0, GPIO_SET_GROUP_INFO = 1, GPIO_GET_VIRT_EN = 2, + GPIO_SET_VIRT_EN = 3, + GPIO_SET_STORE_ST = 4, + GPIO_CLEAR_STORE_ST = 5, }; /* SIP_CPU_PM_CFG child configs */ From c38ef781faa4ccdc539678339002390a97e4333f Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Wed, 11 Jun 2025 19:08:22 +0800 Subject: [PATCH 06/49] ARM: configs: Add rv1126b-tb.config Update by: make ARCH=arm rv1126b_defconfig cp .config tmp.config make ARCH=arm rv1126b_defconfig rv1126b-tb.config make ARCH=arm menuconfig ./scripts/diffconfig -m tmp.config .config > arch/arm/configs/rv1126b-tb.config Signed-off-by: Weiwen Chen Change-Id: Ic684f55c9ffaf0d2c62a6b579e94047e7b69bad3 --- arch/arm/configs/rv1126b-tb.config | 459 +++++++++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 arch/arm/configs/rv1126b-tb.config diff --git a/arch/arm/configs/rv1126b-tb.config b/arch/arm/configs/rv1126b-tb.config new file mode 100644 index 000000000000..8672c83a7b26 --- /dev/null +++ b/arch/arm/configs/rv1126b-tb.config @@ -0,0 +1,459 @@ +CONFIG_BLK_DEV_INITRD=y +# CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND is not set +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y +CONFIG_CRYPTO=y +CONFIG_DAX=y +CONFIG_EROFS_FS=y +# CONFIG_ETHERNET is not set +CONFIG_JFFS2_FS=y +CONFIG_KERNEL_GZIP=y +# CONFIG_KERNEL_XZ is not set +CONFIG_LIBCRC32C=y +CONFIG_MAILBOX=y +# CONFIG_MDIO_DEVICE is not set +CONFIG_MMC=y +CONFIG_MTD_BLOCK=y +# CONFIG_PHYLIB is not set +CONFIG_PHY_ROCKCHIP_CSI2_DPHY=y +CONFIG_ROCKCHIP_DVBM=y +CONFIG_ROCKCHIP_HW_DECOMPRESS=y +CONFIG_ROCKCHIP_MULTI_RGA=y +CONFIG_ROCKCHIP_RAMDISK=y +CONFIG_ROCKCHIP_RGA_PROC_FS=y +CONFIG_ROCKCHIP_THUNDER_BOOT=y +CONFIG_ROCKCHIP_THUNDER_BOOT_DEFER_FREE_MEMBLOCK=y +CONFIG_ROCKCHIP_VENDOR_STORAGE=y +# CONFIG_SLUB_SYSFS is not set +CONFIG_SPI=y +CONFIG_VIDEO_ROCKCHIP_AIISP=y +CONFIG_VIDEO_ROCKCHIP_AVSP=y +CONFIG_VIDEO_ROCKCHIP_CIF=y +CONFIG_VIDEO_ROCKCHIP_ISP=y +CONFIG_VIDEO_ROCKCHIP_VPSS=y +CONFIG_VIDEO_SC200AI=y +CONFIG_VIDEO_SC450AI=y +# CONFIG_AD2S1200 is not set +# CONFIG_AD2S1210 is not set +# CONFIG_AD2S90 is not set +# CONFIG_AD3552R is not set +# CONFIG_AD5360 is not set +# CONFIG_AD5421 is not set +# CONFIG_AD5449 is not set +# CONFIG_AD5504 is not set +# CONFIG_AD5592R is not set +# CONFIG_AD5624R_SPI is not set +# CONFIG_AD5686_SPI is not set +# CONFIG_AD5755 is not set +# CONFIG_AD5758 is not set +# CONFIG_AD5761 is not set +# CONFIG_AD5764 is not set +# CONFIG_AD5766 is not set +# CONFIG_AD5770R is not set +# CONFIG_AD5791 is not set +# CONFIG_AD7124 is not set +# CONFIG_AD7192 is not set +# CONFIG_AD7266 is not set +# CONFIG_AD7280 is not set +# CONFIG_AD7292 is not set +# CONFIG_AD7293 is not set +# CONFIG_AD7298 is not set +# CONFIG_AD7303 is not set +# CONFIG_AD74413R is not set +# CONFIG_AD7476 is not set +# CONFIG_AD7606_IFACE_SPI is not set +# CONFIG_AD7766 is not set +# CONFIG_AD7768_1 is not set +# CONFIG_AD7780 is not set +# CONFIG_AD7791 is not set +# CONFIG_AD7793 is not set +# CONFIG_AD7816 is not set +# CONFIG_AD7887 is not set +# CONFIG_AD7923 is not set +# CONFIG_AD7949 is not set +# CONFIG_AD8366 is not set +# CONFIG_AD8801 is not set +# CONFIG_AD9523 is not set +# CONFIG_AD9832 is not set +# CONFIG_AD9834 is not set +# CONFIG_ADA4250 is not set +# CONFIG_ADF4350 is not set +# CONFIG_ADF4371 is not set +# CONFIG_ADIS16080 is not set +# CONFIG_ADIS16130 is not set +# CONFIG_ADIS16136 is not set +# CONFIG_ADIS16201 is not set +# CONFIG_ADIS16203 is not set +# CONFIG_ADIS16209 is not set +# CONFIG_ADIS16240 is not set +# CONFIG_ADIS16260 is not set +# CONFIG_ADIS16400 is not set +# CONFIG_ADIS16460 is not set +# CONFIG_ADIS16475 is not set +# CONFIG_ADIS16480 is not set +# CONFIG_ADMV1013 is not set +# CONFIG_ADMV4420 is not set +# CONFIG_ADRF6780 is not set +# CONFIG_ADXL313_SPI is not set +# CONFIG_ADXL345_SPI is not set +# CONFIG_ADXL355_SPI is not set +# CONFIG_ADXL367_SPI is not set +# CONFIG_ADXL372_SPI is not set +# CONFIG_ADXRS290 is not set +# CONFIG_ADXRS450 is not set +# CONFIG_AFE4403 is not set +# CONFIG_ALTERA_MBOX is not set +# CONFIG_ARM_MHU is not set +# CONFIG_ARM_MHU_V2 is not set +# CONFIG_ARM_SCPI_PROTOCOL is not set +# CONFIG_AS3935 is not set +# CONFIG_BMA220 is not set +# CONFIG_BMC150_MAGN_SPI is not set +# CONFIG_BMI088_ACCEL is not set +# CONFIG_BMI160_SPI is not set +# CONFIG_CRYPTO_842 is not set +# CONFIG_CRYPTO_ADIANTUM is not set +# CONFIG_CRYPTO_AEGIS128 is not set +# CONFIG_CRYPTO_AES is not set +# CONFIG_CRYPTO_AES_ARM is not set +# CONFIG_CRYPTO_AES_TI is not set +CONFIG_CRYPTO_ALGAPI=y +CONFIG_CRYPTO_ALGAPI2=y +# CONFIG_CRYPTO_ANSI_CPRNG is not set +# CONFIG_CRYPTO_ARIA is not set +# CONFIG_CRYPTO_AUTHENC is not set +# CONFIG_CRYPTO_BLAKE2B is not set +# CONFIG_CRYPTO_BLAKE2S_ARM is not set +# CONFIG_CRYPTO_BLOWFISH is not set +# CONFIG_CRYPTO_CAMELLIA is not set +# CONFIG_CRYPTO_CAST5 is not set +# CONFIG_CRYPTO_CAST6 is not set +# CONFIG_CRYPTO_CBC is not set +# CONFIG_CRYPTO_CCM is not set +# CONFIG_CRYPTO_CFB is not set +# CONFIG_CRYPTO_CHACHA20 is not set +# CONFIG_CRYPTO_CHACHA20POLY1305 is not set +# CONFIG_CRYPTO_CHACHA20_NEON is not set +# CONFIG_CRYPTO_CMAC is not set +# CONFIG_CRYPTO_CRC32 is not set +CONFIG_CRYPTO_CRC32C=y +# CONFIG_CRYPTO_CRCT10DIF is not set +# CONFIG_CRYPTO_CRYPTD is not set +# CONFIG_CRYPTO_CTR is not set +# CONFIG_CRYPTO_CTS is not set +# CONFIG_CRYPTO_CURVE25519 is not set +# CONFIG_CRYPTO_DEFLATE is not set +# CONFIG_CRYPTO_DES is not set +# CONFIG_CRYPTO_DH is not set +# CONFIG_CRYPTO_DRBG_MENU is not set +# CONFIG_CRYPTO_ECB is not set +# CONFIG_CRYPTO_ECDH is not set +# CONFIG_CRYPTO_ECDSA is not set +# CONFIG_CRYPTO_ECHAINIV is not set +# CONFIG_CRYPTO_ECRDSA is not set +# CONFIG_CRYPTO_ESSIV is not set +# CONFIG_CRYPTO_FCRYPT is not set +# CONFIG_CRYPTO_GCM is not set +# CONFIG_CRYPTO_GHASH is not set +CONFIG_CRYPTO_HASH=y +CONFIG_CRYPTO_HASH2=y +# CONFIG_CRYPTO_HCTR2 is not set +# CONFIG_CRYPTO_HMAC is not set +# CONFIG_CRYPTO_HW is not set +# CONFIG_CRYPTO_JITTERENTROPY is not set +# CONFIG_CRYPTO_KEYWRAP is not set +# CONFIG_CRYPTO_LIB_CHACHA20POLY1305 is not set +CONFIG_CRYPTO_LIB_UTILS=y +# CONFIG_CRYPTO_LRW is not set +# CONFIG_CRYPTO_LZ4 is not set +# CONFIG_CRYPTO_LZ4HC is not set +# CONFIG_CRYPTO_LZO is not set +# CONFIG_CRYPTO_MANAGER is not set +CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y +# CONFIG_CRYPTO_MD4 is not set +# CONFIG_CRYPTO_MD5 is not set +# CONFIG_CRYPTO_MICHAEL_MIC is not set +# CONFIG_CRYPTO_NULL is not set +# CONFIG_CRYPTO_OFB is not set +# CONFIG_CRYPTO_PCBC is not set +# CONFIG_CRYPTO_PCRYPT is not set +# CONFIG_CRYPTO_POLY1305 is not set +# CONFIG_CRYPTO_POLY1305_ARM is not set +# CONFIG_CRYPTO_RMD160 is not set +# CONFIG_CRYPTO_RSA is not set +# CONFIG_CRYPTO_SEQIV is not set +# CONFIG_CRYPTO_SERPENT is not set +# CONFIG_CRYPTO_SHA1 is not set +# CONFIG_CRYPTO_SHA1_ARM is not set +# CONFIG_CRYPTO_SHA256 is not set +# CONFIG_CRYPTO_SHA256_ARM is not set +# CONFIG_CRYPTO_SHA3 is not set +# CONFIG_CRYPTO_SHA512 is not set +# CONFIG_CRYPTO_SHA512_ARM is not set +# CONFIG_CRYPTO_SM2 is not set +# CONFIG_CRYPTO_SM3_GENERIC is not set +# CONFIG_CRYPTO_SM4_GENERIC is not set +# CONFIG_CRYPTO_STREEBOG is not set +# CONFIG_CRYPTO_TEST is not set +# CONFIG_CRYPTO_TWOFISH is not set +# CONFIG_CRYPTO_USER is not set +# CONFIG_CRYPTO_USER_API_AEAD is not set +# CONFIG_CRYPTO_USER_API_HASH is not set +# CONFIG_CRYPTO_USER_API_RNG is not set +# CONFIG_CRYPTO_USER_API_SKCIPHER is not set +# CONFIG_CRYPTO_VMAC is not set +# CONFIG_CRYPTO_WP512 is not set +# CONFIG_CRYPTO_XCBC is not set +# CONFIG_CRYPTO_XTS is not set +# CONFIG_CRYPTO_XXHASH is not set +# CONFIG_CRYPTO_ZSTD is not set +# CONFIG_EEPROM_93XX46 is not set +# CONFIG_EEPROM_AT25 is not set +# CONFIG_EROFS_FS_DEBUG is not set +# CONFIG_EROFS_FS_XATTR is not set +# CONFIG_EROFS_FS_ZIP is not set +# CONFIG_EZX_PCAP is not set +CONFIG_FS_DAX=y +CONFIG_FS_IOMAP=y +# CONFIG_FXLS8962AF_SPI is not set +# CONFIG_FXOS8700_SPI is not set +# CONFIG_GPIO_74X164 is not set +# CONFIG_GPIO_MAX3191X is not set +# CONFIG_GPIO_MAX7301 is not set +# CONFIG_GPIO_MC33880 is not set +# CONFIG_GPIO_PISOSR is not set +# CONFIG_GPIO_XRA1403 is not set +# CONFIG_HI8435 is not set +# CONFIG_IIO_SSP_SENSORHUB is not set +CONFIG_INITCALL_ASYNC=y +# CONFIG_INITRAMFS_FORCE is not set +CONFIG_INITRAMFS_SOURCE="" +# CONFIG_INV_ICM42600_SPI is not set +# CONFIG_INV_ICM42670_SPI is not set +# CONFIG_INV_MPU6050_SPI is not set +# CONFIG_JFFS2_COMPRESSION_OPTIONS is not set +CONFIG_JFFS2_FS_DEBUG=0 +# CONFIG_JFFS2_FS_WBUF_VERIFY is not set +CONFIG_JFFS2_FS_WRITEBUFFER=y +# CONFIG_JFFS2_FS_XATTR is not set +CONFIG_JFFS2_RTIME=y +# CONFIG_JFFS2_SUMMARY is not set +CONFIG_JFFS2_ZLIB=y +# CONFIG_LATTICE_ECP3_CONFIG is not set +# CONFIG_LMK04832 is not set +# CONFIG_LTC1660 is not set +# CONFIG_LTC2496 is not set +# CONFIG_LTC2632 is not set +# CONFIG_LTC2688 is not set +# CONFIG_LTC2983 is not set +CONFIG_MAILBOX_POLL_PERIOD_US=y +# CONFIG_MAILBOX_TEST is not set +# CONFIG_MAX1027 is not set +# CONFIG_MAX11100 is not set +# CONFIG_MAX1118 is not set +# CONFIG_MAX11205 is not set +# CONFIG_MAX1241 is not set +# CONFIG_MAX31856 is not set +# CONFIG_MAX31865 is not set +# CONFIG_MAX5481 is not set +# CONFIG_MAX5487 is not set +# CONFIG_MAXIM_THERMOCOUPLE is not set +# CONFIG_MCP320X is not set +# CONFIG_MCP3911 is not set +# CONFIG_MCP41010 is not set +# CONFIG_MCP4131 is not set +# CONFIG_MCP4922 is not set +# CONFIG_MFD_ARIZONA_SPI is not set +# CONFIG_MFD_CPCAP is not set +# CONFIG_MFD_DA9052_SPI is not set +# CONFIG_MFD_INTEL_M10_BMC is not set +# CONFIG_MFD_MC13XXX_SPI is not set +# CONFIG_MFD_OCELOT is not set +# CONFIG_MFD_RK806_SPI is not set +# CONFIG_MFD_RSMU_SPI is not set +# CONFIG_MFD_TPS65912_SPI is not set +# CONFIG_MFD_WM831X_SPI is not set +# CONFIG_MICREL_KS8995MA is not set +# CONFIG_MMA7455_SPI is not set +# CONFIG_MMC_ARMMMCI is not set +CONFIG_MMC_BLOCK=y +CONFIG_MMC_BLOCK_MINORS=32 +# CONFIG_MMC_CQHCI is not set +# CONFIG_MMC_DEBUG is not set +CONFIG_MMC_DW=y +# CONFIG_MMC_DW_BLUEFIELD is not set +# CONFIG_MMC_DW_EXYNOS is not set +# CONFIG_MMC_DW_HI3798CV200 is not set +# CONFIG_MMC_DW_K3 is not set +CONFIG_MMC_DW_PLTFM=y +CONFIG_MMC_DW_ROCKCHIP=m +# CONFIG_MMC_HSQ is not set +# CONFIG_MMC_MTK is not set +CONFIG_MMC_QUEUE_DEPTH=1 +# CONFIG_MMC_SDHCI is not set +# CONFIG_MMC_SPI is not set +# CONFIG_MMC_TEST is not set +# CONFIG_MMC_USDHI6ROL0 is not set +# CONFIG_MOXTET is not set +# CONFIG_MPL115_SPI is not set +CONFIG_MTD_BLKDEVS=y +# CONFIG_MTD_DATAFLASH is not set +# CONFIG_MTD_MCHP23K256 is not set +# CONFIG_MTD_MCHP48L640 is not set +# CONFIG_MTD_SPI_NAND is not set +CONFIG_MTD_SPI_NOR=y +# CONFIG_MTD_SPI_NOR_ATMEL is not set +# CONFIG_MTD_SPI_NOR_BOYA is not set +# CONFIG_MTD_SPI_NOR_CATALYST is not set +# CONFIG_MTD_SPI_NOR_DEVICE_AUTOSELECT is not set +# CONFIG_MTD_SPI_NOR_DOSILICON is not set +# CONFIG_MTD_SPI_NOR_EON is not set +# CONFIG_MTD_SPI_NOR_ESMT is not set +# CONFIG_MTD_SPI_NOR_EVERSPIN is not set +# CONFIG_MTD_SPI_NOR_FMSH is not set +# CONFIG_MTD_SPI_NOR_FUJITSU is not set +CONFIG_MTD_SPI_NOR_GIGADEVICE=y +# CONFIG_MTD_SPI_NOR_INTEL is not set +# CONFIG_MTD_SPI_NOR_ISSI is not set +CONFIG_MTD_SPI_NOR_MACRONIX=y +CONFIG_MTD_SPI_NOR_MISC=y +# CONFIG_MTD_SPI_NOR_NORMEM is not set +# CONFIG_MTD_SPI_NOR_PUYA is not set +# CONFIG_MTD_SPI_NOR_SPANSION is not set +# CONFIG_MTD_SPI_NOR_SST is not set +# CONFIG_MTD_SPI_NOR_STMICRO is not set +# CONFIG_MTD_SPI_NOR_SWP_DISABLE is not set +CONFIG_MTD_SPI_NOR_SWP_DISABLE_ON_VOLATILE=y +# CONFIG_MTD_SPI_NOR_SWP_KEEP is not set +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y +CONFIG_MTD_SPI_NOR_WINBOND=y +# CONFIG_MTD_SPI_NOR_XILINX is not set +# CONFIG_MTD_SPI_NOR_XMC is not set +CONFIG_MTD_SPI_NOR_XTX=y +# CONFIG_MTD_SST25L is not set +# CONFIG_PI433 is not set +# CONFIG_PL320_MBOX is not set +# CONFIG_PLATFORM_MHU is not set +# CONFIG_PWRSEQ_EMMC is not set +CONFIG_PWRSEQ_SIMPLE=y +# CONFIG_RD_BZIP2 is not set +# CONFIG_RD_GZIP is not set +# CONFIG_RD_LZ4 is not set +# CONFIG_RD_LZMA is not set +# CONFIG_RD_LZO is not set +# CONFIG_RD_XZ is not set +# CONFIG_RD_ZSTD is not set +CONFIG_REGMAP_SPI=y +# CONFIG_REGULATOR_TPS6524X is not set +# CONFIG_ROCKCHIP_HW_DECOMPRESS_TEST is not set +CONFIG_ROCKCHIP_MBOX=y +# CONFIG_ROCKCHIP_MBOX_DEMO is not set +# CONFIG_ROCKCHIP_MMC_VENDOR_STORAGE is not set +# CONFIG_ROCKCHIP_MTD_VENDOR_STORAGE is not set +# CONFIG_ROCKCHIP_RAM_VENDOR_STORAGE is not set +CONFIG_ROCKCHIP_RGA_DEBUGGER=y +CONFIG_ROCKCHIP_THUNDER_BOOT_MMC=y +CONFIG_ROCKCHIP_THUNDER_BOOT_SERVICE=y +CONFIG_ROCKCHIP_THUNDER_BOOT_SFC=y +# CONFIG_RPMSG_QCOM_GLINK_RPM is not set +# CONFIG_RPMSG_ROCKCHIP_MBOX is not set +# CONFIG_RTC_DRV_DS1302 is not set +# CONFIG_RTC_DRV_DS1305 is not set +# CONFIG_RTC_DRV_DS1343 is not set +# CONFIG_RTC_DRV_DS1347 is not set +# CONFIG_RTC_DRV_DS1390 is not set +# CONFIG_RTC_DRV_M41T93 is not set +# CONFIG_RTC_DRV_M41T94 is not set +# CONFIG_RTC_DRV_MAX6902 is not set +# CONFIG_RTC_DRV_MAX6916 is not set +# CONFIG_RTC_DRV_MCP795 is not set +# CONFIG_RTC_DRV_PCF2123 is not set +# CONFIG_RTC_DRV_R9701 is not set +# CONFIG_RTC_DRV_RS5C348 is not set +# CONFIG_RTC_DRV_RX4581 is not set +# CONFIG_SCA3000 is not set +# CONFIG_SCA3300 is not set +# CONFIG_SDIO_UART is not set +# CONFIG_SENSORS_HMC5843_SPI is not set +# CONFIG_SENSORS_RM3100_SPI is not set +# CONFIG_SERIAL_MAX3100 is not set +# CONFIG_SERIAL_MAX310X is not set +# CONFIG_SND_SOC_ADAU1372_SPI is not set +# CONFIG_SND_SOC_ADAU1761_SPI is not set +# CONFIG_SND_SOC_AK4104 is not set +# CONFIG_SND_SOC_CS35L41_SPI is not set +# CONFIG_SND_SOC_CS35L45_SPI is not set +# CONFIG_SND_SOC_CS4271_SPI is not set +# CONFIG_SND_SOC_ES8328_SPI is not set +# CONFIG_SND_SOC_PCM179X_SPI is not set +# CONFIG_SND_SOC_PCM186X_SPI is not set +# CONFIG_SND_SOC_PCM3060_SPI is not set +# CONFIG_SND_SOC_PCM3168A_SPI is not set +# CONFIG_SND_SOC_PCM512x_SPI is not set +# CONFIG_SND_SOC_RK3399_GRU_SOUND is not set +# CONFIG_SND_SOC_ROCKCHIP_SPI_CODEC is not set +# CONFIG_SND_SOC_SSM2602_SPI is not set +# CONFIG_SND_SOC_TLV320AIC23_SPI is not set +# CONFIG_SND_SOC_TLV320AIC32X4_SPI is not set +# CONFIG_SND_SOC_TLV320AIC3X_SPI is not set +# CONFIG_SND_SOC_WM8731_SPI is not set +# CONFIG_SND_SOC_WM8770 is not set +# CONFIG_SND_SOC_WM8804_SPI is not set +# CONFIG_SND_SOC_ZL38060 is not set +# CONFIG_SND_SPI is not set +# CONFIG_SPI_ALTERA is not set +# CONFIG_SPI_AMD is not set +# CONFIG_SPI_AXI_SPI_ENGINE is not set +# CONFIG_SPI_BITBANG is not set +# CONFIG_SPI_CADENCE is not set +# CONFIG_SPI_CADENCE_QUADSPI is not set +# CONFIG_SPI_CADENCE_XSPI is not set +# CONFIG_SPI_DEBUG is not set +# CONFIG_SPI_DESIGNWARE is not set +# CONFIG_SPI_FSL_SPI is not set +# CONFIG_SPI_GPIO is not set +# CONFIG_SPI_LOOPBACK_TEST is not set +CONFIG_SPI_MASTER=y +CONFIG_SPI_MEM=y +# CONFIG_SPI_MICROCHIP_CORE is not set +# CONFIG_SPI_MICROCHIP_CORE_QSPI is not set +# CONFIG_SPI_MUX is not set +# CONFIG_SPI_MXIC is not set +# CONFIG_SPI_NXP_FLEXSPI is not set +# CONFIG_SPI_OC_TINY is not set +# CONFIG_SPI_PL022 is not set +# CONFIG_SPI_ROCKCHIP is not set +# CONFIG_SPI_ROCKCHIP_FLEXBUS_FSPI is not set +# CONFIG_SPI_ROCKCHIP_FLEXBUS_SPI is not set +CONFIG_SPI_ROCKCHIP_SFC=y +# CONFIG_SPI_ROCKCHIP_SLAVE is not set +# CONFIG_SPI_SC18IS602 is not set +# CONFIG_SPI_SIFIVE is not set +# CONFIG_SPI_SLAVE is not set +# CONFIG_SPI_SPIDEV is not set +# CONFIG_SPI_TLE62X0 is not set +# CONFIG_SPI_XCOMM is not set +# CONFIG_SPI_XILINX is not set +# CONFIG_SPI_ZYNQMP_GQSPI is not set +# CONFIG_TI_ADC0832 is not set +# CONFIG_TI_ADC084S021 is not set +# CONFIG_TI_ADC108S102 is not set +# CONFIG_TI_ADC12138 is not set +# CONFIG_TI_ADC128S052 is not set +# CONFIG_TI_ADC161S626 is not set +# CONFIG_TI_ADS124S08 is not set +# CONFIG_TI_ADS131E08 is not set +# CONFIG_TI_ADS7950 is not set +# CONFIG_TI_ADS8344 is not set +# CONFIG_TI_ADS8688 is not set +# CONFIG_TI_DAC082S085 is not set +# CONFIG_TI_DAC7311 is not set +# CONFIG_TI_DAC7612 is not set +# CONFIG_TI_TLC4541 is not set +# CONFIG_TI_TSC2046 is not set +# CONFIG_VIDEO_GS1662 is not set +# CONFIG_VIDEO_ROCKCHIP_PREISP is not set +CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_ISP=y +CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_SETUP=y +# CONFIG_VIDEO_S5C73M3 is not set +CONFIG_ZLIB_DEFLATE=y +CONFIG_ZLIB_INFLATE=y From dbdfcd9a9147dcb32671c47db5f09a7a2dd755f8 Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Thu, 5 Jun 2025 18:15:02 +0800 Subject: [PATCH 07/49] arm64: dts: rockchip: Add rv1126b fastboot for emmc/spi nand/spi nor Signed-off-by: Weiwen Chen Change-Id: I3d6cbaef28a2dd41da7932049c1962560c737a64 --- .../dts/rockchip/rv1126b-fastboot-emmc.dtsi | 55 +++++++++++++++++++ .../rockchip/rv1126b-fastboot-spi-nand.dtsi | 32 +++++++++++ .../rockchip/rv1126b-fastboot-spi-nor.dtsi | 39 +++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 arch/arm64/boot/dts/rockchip/rv1126b-fastboot-emmc.dtsi create mode 100644 arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nand.dtsi create mode 100644 arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nor.dtsi diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-emmc.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-emmc.dtsi new file mode 100644 index 000000000000..698e5b2148a4 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-emmc.dtsi @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/ { + ramdisk: ramdisk { + compatible = "rockchip,ramdisk"; + memory-region = <&ramdisk_r>; + }; + + thunder_boot_mmc: thunder-boot-mmc { + compatible = "rockchip,thunder-boot-mmc"; + reg = <0x21470000 0x4000>; + clocks = <&cru HCLK_EMMC>, <&cru CCLK_EMMC>; + clock-names = "biu", "ciu"; + memory-region-src = <&ramdisk_c>; + memory-region-dst = <&ramdisk_r>; + memory-region-idmac = <&mmc_idmac>; + }; +}; + +&emmc { + memory-region-ecsd = <&mmc_ecsd>; + post-power-on-delay-ms = <0>; +}; + +&reserved_memory { + /* Should enable this node if the security feature is enabled, like TA. */ + tee: tee@48400000 { + reg = <0x48400000 0x00800000>; + status = "disabled"; + }; + + mmc_idmac: mmc@48000000 { + reg = <0x48000000 0x00400000>; + }; + + mmc_ecsd: mmc@47FFFE00{ + reg = <0x47FFFE00 0x00001000>; + }; + + ramdisk_r: ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; + }; + + ramdisk_c: ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; + }; +}; + +&hw_decompress { + memory-region = <&ramdisk_c>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nand.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nand.dtsi new file mode 100644 index 000000000000..d31fb7748499 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nand.dtsi @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/ { + ramdisk: ramdisk { + compatible = "rockchip,ramdisk"; + memory-region = <&ramdisk_r>; + }; +}; + +&reserved_memory { + /* Should enable this node if the security feature is enabled, like TA. */ + tee: tee@48400000 { + reg = <0x48400000 0x00800000>; + status = "disabled"; + }; + + ramdisk_r: ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; + }; + + ramdisk_c: ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; + }; +}; + +&hw_decompress { + memory-region = <&ramdisk_c>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nor.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nor.dtsi new file mode 100644 index 000000000000..354484f5b66d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-fastboot-spi-nor.dtsi @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/ { + ramdisk: ramdisk { + compatible = "rockchip,ramdisk"; + memory-region = <&ramdisk_r>; + }; + + thunder_boot_spi_nor: thunder-boot-spi-nor { + compatible = "rockchip,thunder-boot-sfc"; + reg = <0x21460000 0x4000>; + memory-region-src = <&ramdisk_c>; + memory-region-dst = <&ramdisk_r>; + }; +}; + +&reserved_memory { + /* Should enable this node if the security feature is enabled, like TA. */ + tee: tee@48400000 { + reg = <0x48400000 0x00800000>; + status = "disabled"; + }; + + ramdisk_r: ramdisk_r { + reg = <0x48c40000 (20 * 0x00100000)>; + }; + + ramdisk_c: ramdisk_c { + reg = <0x4a040000 (10 * 0x00100000)>; + }; +}; + +&hw_decompress { + memory-region = <&ramdisk_c>; + status = "okay"; +}; From 18114c62fc775b00e42107639c655f351345b82f Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Thu, 5 Jun 2025 18:30:29 +0800 Subject: [PATCH 08/49] arm64: dts: rockchip: Add rv1126b-evb1-v10-fastboot-emmc board support Signed-off-by: Weiwen Chen Change-Id: If71a6db9de1123617c60107ae951190e156c1985 --- arch/arm64/boot/dts/rockchip/Makefile | 1 + .../rv1126b-evb1-v10-fastboot-emmc.dts | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-emmc.dts diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile index 861f281c62c1..92ffd13af907 100644 --- a/arch/arm64/boot/dts/rockchip/Makefile +++ b/arch/arm64/boot/dts/rockchip/Makefile @@ -377,6 +377,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-amp.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-bt-sco.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-dual-4k.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-dv.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-fastboot-emmc.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-spi-nor.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb2-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb2-v10-mcu-k350c4516t.dtb diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-emmc.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-emmc.dts new file mode 100644 index 000000000000..9ee73d5f6811 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-emmc.dts @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +#include "rv1126b-evb1-v10.dts" +#include "rv1126b-fastboot-emmc.dtsi" + +/ { + model = "Rockchip RV1126B EVB1 V10 Arm64 Fastboot Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; +}; From bcfc3501dbada715c2d60844293a73bf7f893a37 Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Thu, 5 Jun 2025 18:33:00 +0800 Subject: [PATCH 09/49] ARM: dts: rockchip: Add rv1126b-evb1-v10-fastboot-emmc board support Signed-off-by: Weiwen Chen Change-Id: I1d2ad01e3571a8e71382cb1f933759176e89adec --- arch/arm/boot/dts/Makefile | 1 + .../dts/rv1126b-evb1-v10-fastboot-emmc.dts | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-emmc.dts diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 2bfcd951e676..de3b13b1e2e7 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -1179,6 +1179,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += \ rv1126b-evb1-v10-bt-sco.dtb \ rv1126b-evb1-v10-dual-4k.dtb \ rv1126b-evb1-v10-dv.dtb \ + rv1126b-evb1-v10-fastboot-emmc.dtb \ rv1126b-evb1-v10-spi-nor.dtb \ rv1126b-evb2-v10.dtb \ rv1126b-evb2-v10-mcu-k350c4516t.dtb \ diff --git a/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-emmc.dts b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-emmc.dts new file mode 100644 index 000000000000..ae214ef870ac --- /dev/null +++ b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-emmc.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +#include "arm64/rockchip/rv1126b-evb1-v10-fastboot-emmc.dts" + +/ { + model = "Rockchip RV1126B EVB1 V10 Fastboot Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; +}; From 2717b456d988ca85bc291168680d1063e1bbac49 Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Fri, 13 Jun 2025 18:56:04 +0800 Subject: [PATCH 10/49] arm64: dts: rockchip: Add rv1126b-evb1-v10-fastboot-spi-nand board Signed-off-by: Weiwen Chen Change-Id: Id6df62c37b02fdedd70e2ef2fec8fd0d31fb8949 --- arch/arm64/boot/dts/rockchip/Makefile | 1 + .../rv1126b-evb1-v10-fastboot-spi-nand.dts | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nand.dts diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile index 92ffd13af907..49e961d51eb5 100644 --- a/arch/arm64/boot/dts/rockchip/Makefile +++ b/arch/arm64/boot/dts/rockchip/Makefile @@ -378,6 +378,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-bt-sco.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-dual-4k.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-dv.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-fastboot-emmc.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-fastboot-spi-nand.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-spi-nor.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb2-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb2-v10-mcu-k350c4516t.dtb diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nand.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nand.dts new file mode 100644 index 000000000000..a9d1046d7ff1 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nand.dts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; +#include "rv1126b-evb1-v10.dts" +#include "rv1126b-fastboot-spi-nand.dtsi" + +/ { + model = "Rockchip RV1126B EVB1 V10 Arm64 Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot-spi-nand", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&fspi0 { + status = "okay"; + + flash@0 { + compatible = "spi-nand"; + reg = <0>; + spi-max-frequency = <75000000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <1>; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; +}; From d52ebcc722f1e286c7335415c1a1c4e85180b53d Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Fri, 13 Jun 2025 18:56:40 +0800 Subject: [PATCH 11/49] ARM: dts: rockchip: Add rv1126b-evb1-v10-fastboot-spi-nand board Signed-off-by: Weiwen Chen Change-Id: Ia1ce2d32c97720f3ce592584cefeba8ea5aad918 --- arch/arm/boot/dts/Makefile | 1 + .../rv1126b-evb1-v10-fastboot-spi-nand.dts | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nand.dts diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index de3b13b1e2e7..ab560d35b674 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -1180,6 +1180,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += \ rv1126b-evb1-v10-dual-4k.dtb \ rv1126b-evb1-v10-dv.dtb \ rv1126b-evb1-v10-fastboot-emmc.dtb \ + rv1126b-evb1-v10-fastboot-spi-nand.dtb \ rv1126b-evb1-v10-spi-nor.dtb \ rv1126b-evb2-v10.dtb \ rv1126b-evb2-v10-mcu-k350c4516t.dtb \ diff --git a/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nand.dts b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nand.dts new file mode 100644 index 000000000000..d96e9b3845bd --- /dev/null +++ b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nand.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +#include "arm64/rockchip/rv1126b-evb1-v10-fastboot-spi-nand.dts" + +/ { + model = "Rockchip RV1126B EVB1 V10 Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot-spi-nand", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (40 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4b440000 (20 * 0x00100000)>; +}; From 7fd0937e714150902d7956fa69d2a59e7f914328 Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Fri, 13 Jun 2025 18:57:29 +0800 Subject: [PATCH 12/49] arm64: dts: rockchip: Add rv1126b-evb1-v10-fastboot-spi-nor board Signed-off-by: Weiwen Chen Change-Id: Ie0faf6fe3da72b3707e620b829e3a474e51f2422 --- arch/arm64/boot/dts/rockchip/Makefile | 1 + .../rv1126b-evb1-v10-fastboot-spi-nor.dts | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nor.dts diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile index 49e961d51eb5..e5d0402f772f 100644 --- a/arch/arm64/boot/dts/rockchip/Makefile +++ b/arch/arm64/boot/dts/rockchip/Makefile @@ -379,6 +379,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-dual-4k.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-dv.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-fastboot-emmc.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-fastboot-spi-nand.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-fastboot-spi-nor.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb1-v10-spi-nor.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb2-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rv1126b-evb2-v10-mcu-k350c4516t.dtb diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nor.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nor.dts new file mode 100644 index 000000000000..3f14ea6c77be --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb1-v10-fastboot-spi-nor.dts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; +#include "rv1126b-evb1-v10.dts" +#include "rv1126b-fastboot-spi-nor.dtsi" + +/ { + model = "Rockchip RV1126B EVB1 V10 Arm64 Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot-spi-nor", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&fspi0 { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <75000000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <1>; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (20 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4a040000 (10 * 0x00100000)>; +}; From bde04f381f8e14898ad3003f446221982a350b8a Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Fri, 13 Jun 2025 18:57:56 +0800 Subject: [PATCH 13/49] ARM: dts: rockchip: Add rv1126b-evb1-v10-fastboot-spi-nor board Signed-off-by: Weiwen Chen Change-Id: Ia4f128a751dfdf669329a73b38a3f9777ecf90e0 --- arch/arm/boot/dts/Makefile | 1 + .../dts/rv1126b-evb1-v10-fastboot-spi-nor.dts | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nor.dts diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index ab560d35b674..a4d0587fada1 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -1181,6 +1181,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += \ rv1126b-evb1-v10-dv.dtb \ rv1126b-evb1-v10-fastboot-emmc.dtb \ rv1126b-evb1-v10-fastboot-spi-nand.dtb \ + rv1126b-evb1-v10-fastboot-spi-nor.dtb \ rv1126b-evb1-v10-spi-nor.dtb \ rv1126b-evb2-v10.dtb \ rv1126b-evb2-v10-mcu-k350c4516t.dtb \ diff --git a/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nor.dts b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nor.dts new file mode 100644 index 000000000000..43e000f8e094 --- /dev/null +++ b/arch/arm/boot/dts/rv1126b-evb1-v10-fastboot-spi-nor.dts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +#include "arm64/rockchip/rv1126b-evb1-v10-fastboot-spi-nor.dts" + +/ { + model = "Rockchip RV1126B EVB1 V10 Board"; + compatible = "rockchip,rv1126b-evb1-v10-fastboot-spi-nand", "rockchip,rv1126b"; + + chosen { + bootargs = "loglevel=0 initcall_debug=0 earlycon=uart8250,mmio32,0x20810000 console=ttyFIQ0 root=/dev/rd0 rootfstype=erofs rootflags=dax snd_soc_core.prealloc_buffer_size_kbytes=16 coherent_pool=32K"; + }; +}; + +&ramdisk_r { + reg = <0x48c40000 (20 * 0x00100000)>; +}; + +&ramdisk_c { + reg = <0x4a040000 (10 * 0x00100000)>; +}; From f8b265e116ce95276c724a8dbb6ebc81eb991369 Mon Sep 17 00:00:00 2001 From: LongChang Ma Date: Fri, 13 Jun 2025 17:04:18 +0800 Subject: [PATCH 14/49] media: i2c: fix sc450ai miss config global_reg_list Signed-off-by: LongChang Ma Change-Id: I71ad40a2b49114ab713eb199f8c1f423b9d6c1c5 --- drivers/media/i2c/sc450ai.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/media/i2c/sc450ai.c b/drivers/media/i2c/sc450ai.c index 17794f7ed050..443ebbbbeb14 100644 --- a/drivers/media/i2c/sc450ai.c +++ b/drivers/media/i2c/sc450ai.c @@ -1217,6 +1217,7 @@ static const struct sc450ai_mode supported_modes_2lane[] = { .hts_def = 0x2ee * 4, .vts_def = 0x0638, .bus_fmt = MEDIA_BUS_FMT_SBGGR10_1X10, + .global_reg_list = sc450ai_global_regs_2lane, .reg_list = sc450ai_linear_10_2688x1520_30fps_2lane_regs, .hdr_mode = NO_HDR, .mclk = 27000000, From a5af81fd98c7b5eeaa6d8482e625e62175f5f049 Mon Sep 17 00:00:00 2001 From: Yuefu Su Date: Fri, 13 Jun 2025 09:46:59 +0800 Subject: [PATCH 15/49] arm64: dts: rockchip: rv1126b-thunder-boot: add thunder-boot-service/mbox/thunder-boot-isp Signed-off-by: Yuefu Su Change-Id: Ia7225c269fa2eb065f2de847df581bb555ff5225 --- .../dts/rockchip/rv1126b-thunder-boot.dtsi | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot.dtsi index 890f84f832d1..dc1defc3ab39 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot.dtsi +++ b/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot.dtsi @@ -8,6 +8,49 @@ compatible = "rockchip,ramdisk"; memory-region = <&ramdisk_r>; }; + + thunder_boot_rkisp: thunder-boot-rkisp { + compatible = "rockchip,thunder-boot-rkisp"; + clocks = <&cru CLK_HPMCU>, + <&cru PCLK_HPMCU_INTMUX>, <&cru PCLK_HPMCU_MAILBOX>, + <&cru PCLK_WDT_HPMCU>, <&cru HCLK_CACHE>, + <&cru PCLK_TIMER>, + <&cru ACLK_ISP>, <&cru HCLK_ISP>, + <&cru CLK_CORE_ISP>, <&cru ISP0CLK_VICAP>, + <&cru ACLK_VICAP>, <&cru HCLK_VICAP>, + <&cru DCLK_VICAP>, <&cru ISP0CLK_VICAP>, + <&cru PCLK_CSI2HOST0>, <&cru PCLK_CSI2HOST1>, + <&cru CLK_I2C4>, <&cru PCLK_I2C4>; + clock-names = "clk_hpmcu", + "pclk_hpmcu_intmux", "pclk_hpmcu_mailbox", + "pclk_wdt_hpmcu", "hclk_cache", + "pclk_timer", + "aclk_isp", "hclk_isp", + "clk_isp_core", "clk_isp_core_vicap", + "aclk_cif","hclk_cif", + "dclk_cif", "isp0clk_cif", + "pclk_csi2host0", "pclk_csi2host1", + "clk_i2c4", "pclk_i2c4"; + }; + + thunder_boot_service: thunder-boot-service { + compatible = "rockchip,thunder-boot-service"; + mbox-names = "amp-rx"; + mboxes = <&hpmcu_mbox0 0>; + resets = <&cru SRST_RESETN_HPMCU_FULL_CLUSTER>; + reset-names = "hpmcu_full_cluster"; + memory-region = <&rtos>; + status = "okay"; + }; +}; + +&hpmcu_mbox0 { + status = "okay"; +}; + +&hw_decompress { + memory-region = <&ramdisk_c>; + status = "okay"; }; &reserved_memory { @@ -88,8 +131,3 @@ reg = <0x49640000 (5 * 0x00100000)>; }; }; - -&hw_decompress { - memory-region = <&ramdisk_c>; - status = "okay"; -}; From 97c68a63993cfef04c698ef95206c70de3034f9a Mon Sep 17 00:00:00 2001 From: Yuefu Su Date: Fri, 13 Jun 2025 18:46:08 +0800 Subject: [PATCH 16/49] arm64: dts: rockchip: Add rv1126b-evb2-v10.dtsi Signed-off-by: Yuefu Su Change-Id: I06d91a98be2c1934876f58ffde8a48b7aa805253 --- .../boot/dts/rockchip/rv1126b-evb2-v10.dtsi | 467 ++++++++++++++++++ 1 file changed, 467 insertions(+) create mode 100644 arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dtsi diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dtsi new file mode 100644 index 000000000000..7f9cbcbc70b8 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dtsi @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +/ { + gpio-keys { + compatible = "gpio-keys"; + autorepeat; + pinctrl-names = "default"; + pinctrl-0 = <&pwr_key>; + power-key { + gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>; + linux,code = ; + label ="GPIO Key Power"; + debounce-interval = <100>; + wakeup-source; + /* gpio-key,wakeup; */ + }; + }; + + vcc5v0_dcin: vcc5v0-dcin { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_dcin"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vcc12v_dcin>; + }; + + vcc5v0_host: vcc5v0-host { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_host"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&gpio0 RK_PD1 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc5v0_dcin>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_host_en>; + }; + + vccsys_stb: vccsys-stb { + compatible = "regulator-fixed"; + regulator-name = "vccsys_stb"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vcc5v0_dcin>; + }; + + vcc3v3_stb: vcc3v3-stb { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_stb"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vccsys_stb>; + }; + + vcc1v8_pmu: vdd1_1v8_ddr: vcc1v8_pmu { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_stb"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vccsys_stb>; + }; + + vdd2_1v1_ddr: vdd2_1v1_ddr { + compatible = "regulator-fixed"; + regulator-name = "vdd2_1v1_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + vin-supply = <&vccsys_stb>; + }; + + vddq_0v6_lp4x: vddq-0v6-lp4x { + compatible = "regulator-fixed"; + regulator-name = "vddq_0v6_lp4x"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <600000>; + regulator-max-microvolt = <600000>; + vin-supply = <&vccsys_stb>; + }; + + vcc3v3_pmu: vcc3v3-pmu { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_pmu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vccsys_stb>; + }; + + vcc0v9_pmu: vcc0v9-pmu { + compatible = "regulator-fixed"; + regulator-name = "vcc0v9_pmu"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc1v8_pmu>; + }; + + vcc_mipi: vcc-mipi{ + compatible = "regulator-fixed"; + regulator-name = "vcc_mipi"; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc1v8_pmu>; + }; + + vcc_1v8: vcc-1v8 { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vccsys_stb>; + }; + + vcc_3v3: vcc3v3_dev: vcc-3v3 { + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vccsys_stb>; + }; + + vcc_lcd: vcc-lcd { + compatible = "regulator-fixed"; + gpio = <&gpio7 RK_PA7 GPIO_ACTIVE_HIGH>; + regulator-name = "vcc_lcd"; + enable-active-high; + }; + + vcc_sd: vcc-sd { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PB0 GPIO_ACTIVE_LOW>; + regulator-name = "vcc_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + enable-active-low; + vin-supply = <&vcc_3v3>; + }; + + vccio_sd: vccio-sd { + compatible = "regulator-gpio"; + regulator-boot-on; + regulator-name = "vccio_sd"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_volt>; + gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + vin-supply = <&vccsys_stb>; + states = <1800000 0x0 + 3300000 0x1>; + }; + + vdd_log: vdd-log { + compatible = "pwm-regulator"; + pwms = <&pwm0_8ch_2 0 25000 1>; + regulator-name = "vdd_log"; + regulator-init-microvolt = <905000>; + regulator-min-microvolt = <810000>; + regulator-max-microvolt = <1006000>; + regulator-always-on; + regulator-boot-on; + pwm-supply = <&vccsys_stb>; + status = "okay"; + }; + + vdd_cpu: vdd-cpu { + compatible = "pwm-regulator"; + pwms = <&pwm0_8ch_0 0 25000 1>; + regulator-name = "vdd_cpu"; + regulator-init-microvolt = <950000>; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1100000>; + regulator-always-on; + regulator-boot-on; + pwm-supply = <&vccsys_stb>; + status = "okay"; + }; + + vdd_npu: vdd-npu { + compatible = "pwm-regulator"; + pwms = <&pwm0_8ch_1 0 25000 1>; + regulator-name = "vdd_npu"; + regulator-init-microvolt = <950000>; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1100000>; + regulator-always-on; + regulator-boot-on; + pwm-supply = <&vccsys_stb>; + status = "okay"; + }; + + wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart2m0_rtsn_pins>; + pinctrl-1 = <&uart2_gpios>; + BT,power_gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless-wlan { + compatible = "wlan-platdata"; + rockchip,grf = <&grf>; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_wake_host>; + wifi_chip_type = "rk96x"; + WIFI,host_wake_irq = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&acdcdig_dsm { + pa-ctl-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&acodec_sound { + status = "okay"; +}; + +&audio_codec { + status = "okay"; +}; + +&backlight { + pwms = <&pwm2_8ch_7 0 25000 0>; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&display_subsystem { + status = "okay"; +}; + +&dsi { + status = "okay"; +}; + +&dsi_in_vop { + status = "okay"; +}; + +&dsi_panel { + power-supply = <&vcc_lcd>; +}; + +&fspi0 { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <100000000>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <1>; + }; +}; + +&gmac { + phy-mode = "rmii"; + clock_in_out = "input"; + phy-handle = <&rmii_phy>; + status = "okay"; +}; + +&i2c2 { + pinctrl-0 = <&i2c2m1_pins>; + status = "okay"; + + gt1x: gt1x@14 { + compatible = "goodix,gt1x"; + reg = <0x14>; + pinctrl-names = "default"; + pinctrl-0 = <&touch_gpio>; + vdd_ana-supply = <&vcc_lcd>; + goodix,rst-gpio = <&gpio5 RK_PD6 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio3 RK_PB7 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +&mdio { + rmii_phy: ethernet-phy@2 { + compatible = "ethernet-phy-id0680.8101", "ethernet-phy-ieee802.3-c22"; + reg = <2>; + clocks = <&cru CLK_MACPHY>; + clock-frequency = <24000000>; + resets = <&cru SRST_RESETN_MACPHY>; + pinctrl-names = "default"; + pinctrl-0 = <&fephym2_pins>; + phy-is-integrated; + }; +}; + +&mipi_dphy { + status = "okay"; +}; + +&pinctrl { + buttons { + pwr_key: pwr-key { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdmmc { + /omit-if-no-ref/ + sdmmc_volt: sdmmc-volt { + rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + touch { + touch_gpio: touch-gpio { + rockchip,pins = + <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>, + <5 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb { + vcc5v0_host_en: vcc5v0-host-en { + rockchip,pins = <0 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-bluetooth { + uart2_gpios: uart2-gpios { + rockchip,pins = <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wireless-wlan { + wifi_wake_host: wifi-wake-host { + rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pwm0_8ch_0 { + status = "okay"; +}; + +&pwm0_8ch_1 { + status = "okay"; +}; + +&pwm0_8ch_2 { + status = "okay"; +}; + +&pwm2_8ch_7 { + status = "okay"; +}; + +&rkaiisp { + status = "okay"; +}; + +&rkaiisp_mmu { + status = "okay"; +}; + +&rkaiisp_vir0 { + status = "okay"; +}; + +&rknpu { + rknpu-supply = <&vdd_npu>; +}; + +&route_dsi { + status = "okay"; +}; + +&rtc { + rockchip,rtc-suspend-bypass; + status = "okay"; +}; + +&sai2 { + rockchip,sai-rx-route = <1 0 2 3>; + status = "okay"; + /delete-property/ pinctrl-names; + /delete-property/ pinctrl-0; +}; + +&saradc0 { + vref-supply = <&vcc_1v8>; +}; + +&sdmmc0 { + max-frequency = <200000000>; + no-sdio; + no-mmc; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + disable-wp; + sd-uhs-sdr104; + vmmc-supply = <&vcc_sd>; + vqmmc-supply = <&vccio_sd>; + status = "okay"; +}; + +&sdmmc1 { + bus-width = <1>; + cap-sd-highspeed; + no-sd; + no-mmc; + max-frequency = <200000000>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1_clk_pins &sdmmc1_cmd_pins &sdmmc1_bus4_pins>; + keep-power-in-suspend; + non-removable; + //mmc-pwrseq = <&sdio_pwrseq>; + //sd-uhs-sdr104; + status = "okay"; +}; + +&uart2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart2m0_xfer_pins &uart2m0_ctsn_pins>; +}; + +&usb2phy_host { + phy-supply = <&vcc5v0_host>; +}; + +&usb3phy { + status = "disabled"; +}; + +&usb_drd_dwc3 { + dr_mode = "otg"; + extcon = <&usb2phy>; + maximum-speed = "high-speed"; + phys = <&usb2phy_otg>; + phy-names = "usb2-phy"; + snps,dis_u2_susphy_quirk; + snps,usb2-lpm-disable; +}; From 404d19120e5e0619226e18967a5c299b23a179a1 Mon Sep 17 00:00:00 2001 From: Yuefu Su Date: Fri, 13 Jun 2025 18:47:00 +0800 Subject: [PATCH 17/49] arm64: dts: rockchip: rv1126b-evb2-v10: Move some nodes to rv1126b-evb2-v10.dtsi Signed-off-by: Yuefu Su Change-Id: I217660e8281fb7ea68bc78878f9af17bf7a921d9 --- .../boot/dts/rockchip/rv1126b-evb2-v10.dts | 462 +----------------- 1 file changed, 1 insertion(+), 461 deletions(-) diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dts index d1ceaa6832c6..96dd2a1089d5 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10.dts @@ -6,405 +6,18 @@ /dts-v1/; #include "rv1126b.dtsi" #include "rv1126b-evb.dtsi" +#include "rv1126b-evb2-v10.dtsi" #include "rv1126b-evb-cam-csi0.dtsi" / { model = "Rockchip RV1126B EVB2 V10 Board"; compatible = "rockchip,rv1126b-evb2-v10", "rockchip,rv1126b"; - - gpio-keys { - compatible = "gpio-keys"; - autorepeat; - pinctrl-names = "default"; - pinctrl-0 = <&pwr_key>; - power-key { - gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>; - linux,code = ; - label ="GPIO Key Power"; - debounce-interval = <100>; - wakeup-source; - /* gpio-key,wakeup; */ - }; - }; - - vcc5v0_dcin: vcc5v0-dcin { - compatible = "regulator-fixed"; - regulator-name = "vcc5v0_dcin"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - vin-supply = <&vcc12v_dcin>; - }; - - vcc5v0_host: vcc5v0-host { - compatible = "regulator-fixed"; - regulator-name = "vcc5v0_host"; - regulator-boot-on; - regulator-always-on; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - enable-active-high; - gpio = <&gpio0 RK_PD1 GPIO_ACTIVE_HIGH>; - vin-supply = <&vcc5v0_dcin>; - pinctrl-names = "default"; - pinctrl-0 = <&vcc5v0_host_en>; - }; - - vccsys_stb: vccsys-stb { - compatible = "regulator-fixed"; - regulator-name = "vccsys_stb"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - vin-supply = <&vcc5v0_dcin>; - }; - - vcc3v3_stb: vcc3v3-stb { - compatible = "regulator-fixed"; - regulator-name = "vcc3v3_stb"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - vin-supply = <&vccsys_stb>; - }; - - vcc1v8_pmu: vdd1_1v8_ddr: vcc1v8_pmu { - compatible = "regulator-fixed"; - regulator-name = "vcc3v3_stb"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - vin-supply = <&vccsys_stb>; - }; - - vdd2_1v1_ddr: vdd2_1v1_ddr { - compatible = "regulator-fixed"; - regulator-name = "vdd2_1v1_ddr"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <1100000>; - regulator-max-microvolt = <1100000>; - vin-supply = <&vccsys_stb>; - }; - - vddq_0v6_lp4x: vddq-0v6-lp4x { - compatible = "regulator-fixed"; - regulator-name = "vddq_0v6_lp4x"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <600000>; - vin-supply = <&vccsys_stb>; - }; - - vcc3v3_pmu: vcc3v3-pmu { - compatible = "regulator-fixed"; - regulator-name = "vcc3v3_pmu"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - vin-supply = <&vccsys_stb>; - }; - - vcc0v9_pmu: vcc0v9-pmu { - compatible = "regulator-fixed"; - regulator-name = "vcc0v9_pmu"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <900000>; - regulator-max-microvolt = <900000>; - vin-supply = <&vcc1v8_pmu>; - }; - - vcc_mipi: vcc-mipi{ - compatible = "regulator-fixed"; - regulator-name = "vcc_mipi"; - regulator-always-on; - regulator-boot-on; - vin-supply = <&vcc1v8_pmu>; - }; - - vcc_1v8: vcc-1v8 { - compatible = "regulator-fixed"; - regulator-name = "vcc_1v8"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - vin-supply = <&vccsys_stb>; - }; - - vcc_3v3: vcc3v3_dev: vcc-3v3 { - compatible = "regulator-fixed"; - regulator-name = "vcc_3v3"; - regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - vin-supply = <&vccsys_stb>; - }; - - vcc_lcd: vcc-lcd { - compatible = "regulator-fixed"; - gpio = <&gpio7 RK_PA7 GPIO_ACTIVE_HIGH>; - regulator-name = "vcc_lcd"; - enable-active-high; - }; - - vcc_sd: vcc-sd { - compatible = "regulator-fixed"; - gpio = <&gpio0 RK_PB0 GPIO_ACTIVE_LOW>; - regulator-name = "vcc_sd"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - enable-active-low; - vin-supply = <&vcc_3v3>; - }; - - vccio_sd: vccio-sd { - compatible = "regulator-gpio"; - regulator-boot-on; - regulator-name = "vccio_sd"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <3300000>; - pinctrl-names = "default"; - pinctrl-0 = <&sdmmc_volt>; - gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; - vin-supply = <&vccsys_stb>; - states = <1800000 0x0 - 3300000 0x1>; - }; - - vdd_log: vdd-log { - compatible = "pwm-regulator"; - pwms = <&pwm0_8ch_2 0 25000 1>; - regulator-name = "vdd_log"; - regulator-init-microvolt = <905000>; - regulator-min-microvolt = <810000>; - regulator-max-microvolt = <1006000>; - regulator-always-on; - regulator-boot-on; - pwm-supply = <&vccsys_stb>; - status = "okay"; - }; - - vdd_cpu: vdd-cpu { - compatible = "pwm-regulator"; - pwms = <&pwm0_8ch_0 0 25000 1>; - regulator-name = "vdd_cpu"; - regulator-init-microvolt = <950000>; - regulator-min-microvolt = <750000>; - regulator-max-microvolt = <1100000>; - regulator-always-on; - regulator-boot-on; - pwm-supply = <&vccsys_stb>; - status = "okay"; - }; - - vdd_npu: vdd-npu { - compatible = "pwm-regulator"; - pwms = <&pwm0_8ch_1 0 25000 1>; - regulator-name = "vdd_npu"; - regulator-init-microvolt = <950000>; - regulator-min-microvolt = <750000>; - regulator-max-microvolt = <1100000>; - regulator-always-on; - regulator-boot-on; - pwm-supply = <&vccsys_stb>; - status = "okay"; - }; - - wireless-bluetooth { - compatible = "bluetooth-platdata"; - uart_rts_gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>; - pinctrl-names = "default", "rts_gpio"; - pinctrl-0 = <&uart2m0_rtsn_pins>; - pinctrl-1 = <&uart2_gpios>; - BT,power_gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; - status = "okay"; - }; - - wireless-wlan { - compatible = "wlan-platdata"; - rockchip,grf = <&grf>; - pinctrl-names = "default"; - pinctrl-0 = <&wifi_wake_host>; - wifi_chip_type = "rk96x"; - WIFI,host_wake_irq = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; - status = "okay"; - }; -}; - -&acdcdig_dsm { - pa-ctl-gpios = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>; - status = "okay"; -}; - -&acodec_sound { - status = "okay"; -}; - -&audio_codec { - status = "okay"; -}; - -&backlight { - pwms = <&pwm2_8ch_7 0 25000 0>; -}; - -&cpu0 { - cpu-supply = <&vdd_cpu>; -}; - -&display_subsystem { - status = "okay"; -}; - -&dsi { - status = "okay"; -}; - -&dsi_in_vop { - status = "okay"; -}; - -&dsi_panel { - power-supply = <&vcc_lcd>; -}; - -&fspi0 { - status = "okay"; - - flash@0 { - compatible = "jedec,spi-nor"; - reg = <0>; - spi-max-frequency = <100000000>; - spi-rx-bus-width = <4>; - spi-tx-bus-width = <1>; - }; -}; - -&gmac { - phy-mode = "rmii"; - clock_in_out = "input"; - phy-handle = <&rmii_phy>; - status = "okay"; }; &imx415 { reset-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>; }; -&i2c2 { - pinctrl-0 = <&i2c2m1_pins>; - status = "okay"; - - gt1x: gt1x@14 { - compatible = "goodix,gt1x"; - reg = <0x14>; - pinctrl-names = "default"; - pinctrl-0 = <&touch_gpio>; - vdd_ana-supply = <&vcc_lcd>; - goodix,rst-gpio = <&gpio5 RK_PD6 GPIO_ACTIVE_HIGH>; - goodix,irq-gpio = <&gpio3 RK_PB7 IRQ_TYPE_LEVEL_LOW>; - }; -}; - -&mdio { - rmii_phy: ethernet-phy@2 { - compatible = "ethernet-phy-id0680.8101", "ethernet-phy-ieee802.3-c22"; - reg = <2>; - clocks = <&cru CLK_MACPHY>; - clock-frequency = <24000000>; - resets = <&cru SRST_RESETN_MACPHY>; - pinctrl-names = "default"; - pinctrl-0 = <&fephym2_pins>; - phy-is-integrated; - }; -}; - -&mipi_dphy { - status = "okay"; -}; - -&pinctrl { - buttons { - pwr_key: pwr-key { - rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>; - }; - }; - - sdmmc { - /omit-if-no-ref/ - sdmmc_volt: sdmmc-volt { - rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - - touch { - touch_gpio: touch-gpio { - rockchip,pins = - <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_up>, - <5 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - - usb { - vcc5v0_host_en: vcc5v0-host-en { - rockchip,pins = <0 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - - wireless-bluetooth { - uart2_gpios: uart2-gpios { - rockchip,pins = <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - - wireless-wlan { - wifi_wake_host: wifi-wake-host { - rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>; - }; - }; -}; - -&pwm0_8ch_0 { - status = "okay"; -}; - -&pwm0_8ch_1 { - status = "okay"; -}; - -&pwm0_8ch_2 { - status = "okay"; -}; - -&pwm2_8ch_7 { - status = "okay"; -}; - -&rkaiisp { - status = "okay"; -}; - -&rkaiisp_mmu { - status = "okay"; -}; - -&rkaiisp_vir0 { - status = "okay"; -}; - -&rknpu { - rknpu-supply = <&vdd_npu>; -}; - &rockchip_suspend { status = "okay"; @@ -562,26 +175,6 @@ >; }; -&route_dsi { - status = "okay"; -}; - -&rtc { - rockchip,rtc-suspend-bypass; - status = "okay"; -}; - -&sai2 { - rockchip,sai-rx-route = <1 0 2 3>; - status = "okay"; - /delete-property/ pinctrl-names; - /delete-property/ pinctrl-0; -}; - -&saradc0 { - vref-supply = <&vcc_1v8>; -}; - &sc450ai { reset-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_LOW>; }; @@ -590,56 +183,3 @@ reset-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>; rockchip,camera-module-stb = <1>; }; - -&sdmmc0 { - max-frequency = <200000000>; - no-sdio; - no-mmc; - bus-width = <4>; - cap-mmc-highspeed; - cap-sd-highspeed; - disable-wp; - sd-uhs-sdr104; - vmmc-supply = <&vcc_sd>; - vqmmc-supply = <&vccio_sd>; - status = "okay"; -}; - -&sdmmc1 { - bus-width = <1>; - cap-sd-highspeed; - no-sd; - no-mmc; - max-frequency = <200000000>; - pinctrl-names = "default"; - pinctrl-0 = <&sdmmc1_clk_pins &sdmmc1_cmd_pins &sdmmc1_bus4_pins>; - keep-power-in-suspend; - non-removable; - //mmc-pwrseq = <&sdio_pwrseq>; - //sd-uhs-sdr104; - status = "okay"; -}; - -&uart2 { - status = "okay"; - pinctrl-names = "default"; - pinctrl-0 = <&uart2m0_xfer_pins &uart2m0_ctsn_pins>; -}; - -&usb2phy_host { - phy-supply = <&vcc5v0_host>; -}; - -&usb3phy { - status = "disabled"; -}; - -&usb_drd_dwc3 { - dr_mode = "otg"; - extcon = <&usb2phy>; - maximum-speed = "high-speed"; - phys = <&usb2phy_otg>; - phy-names = "usb2-phy"; - snps,dis_u2_susphy_quirk; - snps,usb2-lpm-disable; -}; From 209629d7d4435675ec235ee3c4d4565764db9fa6 Mon Sep 17 00:00:00 2001 From: Finley Xiao Date: Thu, 12 Jun 2025 11:14:37 +0800 Subject: [PATCH 18/49] clk: rockchip: rv1126b: mark tsadc clk as critical Change-Id: I0a999d768793b40bac83f8ce5b1f5328d8a9b8d2 Signed-off-by: Finley Xiao --- drivers/clk/rockchip/clk-rv1126b.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/clk/rockchip/clk-rv1126b.c b/drivers/clk/rockchip/clk-rv1126b.c index 1bff71eedba1..b27cc612a447 100644 --- a/drivers/clk/rockchip/clk-rv1126b.c +++ b/drivers/clk/rockchip/clk-rv1126b.c @@ -924,9 +924,9 @@ static struct rockchip_clk_branch rv1126b_clk_branches[] __initdata = { RV1126B_BUSCLKGATE_CON(4), 14, GFLAGS), GATE(PCLK_UART7, "pclk_uart7", "pclk_bus_root", 0, RV1126B_BUSCLKGATE_CON(4), 15, GFLAGS), - GATE(PCLK_TSADC, "pclk_tsadc", "pclk_bus_root", 0, + GATE(PCLK_TSADC, "pclk_tsadc", "pclk_bus_root", CLK_IS_CRITICAL, RV1126B_BUSCLKGATE_CON(5), 0, GFLAGS), - GATE(CLK_TSADC, "clk_tsadc", "xin24m", 0, + GATE(CLK_TSADC, "clk_tsadc", "xin24m", CLK_IS_CRITICAL, RV1126B_BUSCLKGATE_CON(5), 1, GFLAGS), GATE(HCLK_SAI0, "hclk_sai0", "hclk_bus_root", 0, RV1126B_BUSCLKGATE_CON(5), 2, GFLAGS), @@ -962,7 +962,7 @@ static struct rockchip_clk_branch rv1126b_clk_branches[] __initdata = { RV1126B_BUSCLKGATE_CON(6), 4, GFLAGS), GATE(PCLK_OTP_MASK, "pclk_otp_mask", "pclk_bus_root", 0, RV1126B_BUSCLKGATE_CON(6), 6, GFLAGS), - GATE(CLK_TSADC_PHYCTRL, "clk_tsadc_phyctrl", "xin24m", 0, + GATE(CLK_TSADC_PHYCTRL, "clk_tsadc_phyctrl", "xin24m", CLK_IS_CRITICAL, RV1126B_BUSCLKGATE_CON(6), 8, GFLAGS), MUX(LRCK_SRC_ASRC0, "lrck_src_asrc0", lrck_src_asrc_p, 0, RV1126B_BUSCLKSEL_CON(3), 0, 3, MFLAGS), From 70e221d140ef7b25d732ea31fa302fd33e5d7552 Mon Sep 17 00:00:00 2001 From: Finley Xiao Date: Thu, 12 Jun 2025 11:42:29 +0800 Subject: [PATCH 19/49] thermal: rockchip: Make reset optional Change-Id: I057a53e65d083a8ea24f0727bd0f73545482f2de Signed-off-by: Finley Xiao --- drivers/thermal/rockchip_thermal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c index 0defdbc3ff59..7b8746404a9f 100644 --- a/drivers/thermal/rockchip_thermal.c +++ b/drivers/thermal/rockchip_thermal.c @@ -2687,7 +2687,7 @@ static int rockchip_thermal_probe(struct platform_device *pdev) if (IS_ERR(thermal->regs)) return PTR_ERR(thermal->regs); - thermal->reset = devm_reset_control_array_get(&pdev->dev, false, false); + thermal->reset = devm_reset_control_array_get_optional_exclusive(&pdev->dev); if (IS_ERR(thermal->reset)) { if (PTR_ERR(thermal->reset) != -EPROBE_DEFER) dev_err(&pdev->dev, "failed to get tsadc reset lines\n"); From 5fe15ec28a85e24ab1561c97ca3006c16d1dfbbc Mon Sep 17 00:00:00 2001 From: Finley Xiao Date: Thu, 12 Jun 2025 12:12:45 +0800 Subject: [PATCH 20/49] arm64: dts: rockchip: rv1126b: Remove tsadc reset Change-Id: Id18de9f23b7ae4565c788c8106826e72135a925e Signed-off-by: Finley Xiao --- arch/arm64/boot/dts/rockchip/rv1126b.dtsi | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm64/boot/dts/rockchip/rv1126b.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b.dtsi index aa31538be2e7..a75da64b8ba6 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b.dtsi +++ b/arch/arm64/boot/dts/rockchip/rv1126b.dtsi @@ -1904,9 +1904,6 @@ clocks = <&cru CLK_TSADC>, <&cru PCLK_TSADC>, <&cru CLK_TSADC_PHYCTRL>; clock-names = "tsadc", "apb_pclk", "tsadc_phyctrl"; - resets = <&cru SRST_RESETN_TSADC>, <&cru SRST_PRESETN_TSADC>, - <&cru SRST_RESETN_TSADC_PHYCTRL>; - reset-names = "tsadc", "tsadc-apb", "tsadc-phy"; #thermal-sensor-cells = <1>; rockchip,grf = <&grf>; rockchip,hw-tshut-temp = <120000>; From 84cd2182e7aa10718e1000c32d506dec759cfbb0 Mon Sep 17 00:00:00 2001 From: Finley Xiao Date: Fri, 13 Jun 2025 14:31:59 +0800 Subject: [PATCH 21/49] clk: rockchip: rv1126b: Mark clk timer5 as critical Change-Id: I9cdb9dfae6f9004101a128198c105aabde25fc54 Signed-off-by: Finley Xiao --- drivers/clk/rockchip/clk-rv1126b.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clk/rockchip/clk-rv1126b.c b/drivers/clk/rockchip/clk-rv1126b.c index b27cc612a447..96b939b63a34 100644 --- a/drivers/clk/rockchip/clk-rv1126b.c +++ b/drivers/clk/rockchip/clk-rv1126b.c @@ -866,7 +866,7 @@ static struct rockchip_clk_branch rv1126b_clk_branches[] __initdata = { RV1126B_BUSCLKGATE_CON(2), 14, GFLAGS), GATE(HCLK_RKRNG_NS, "hclk_rkrng_ns", "hclk_rkrng_s_ns", 0, RV1126B_BUSCLKGATE_CON(2), 15, GFLAGS), - GATE(CLK_TIMER5, "clk_timer5", "clk_timer_root", 0, + GATE(CLK_TIMER5, "clk_timer5", "clk_timer_root", CLK_IS_CRITICAL, RV1126B_BUSCLKGATE_CON(2), 11, GFLAGS), GATE(PCLK_I2C0, "pclk_i2c0", "pclk_bus_root", 0, RV1126B_BUSCLKGATE_CON(3), 0, GFLAGS), From 8ac7c4966017e0a947f32edb10290ad9bee2452a Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Wed, 11 Jun 2025 15:51:45 +0800 Subject: [PATCH 22/49] ARM: configs: Update rv1126b-fastboot Signed-off-by: Weiwen Chen Change-Id: I8d4b7250baa2edbe56a155702f38bc700ba9fda7 --- arch/arm/configs/rv1126b-fastboot.config | 112 +++++++++++++++++++---- 1 file changed, 93 insertions(+), 19 deletions(-) diff --git a/arch/arm/configs/rv1126b-fastboot.config b/arch/arm/configs/rv1126b-fastboot.config index 0a84152f88f2..3cb56be2c5e0 100644 --- a/arch/arm/configs/rv1126b-fastboot.config +++ b/arch/arm/configs/rv1126b-fastboot.config @@ -1,45 +1,54 @@ CONFIG_BLK_DEV_INITRD=y -CONFIG_CRC16=m +CONFIG_CRC16=y CONFIG_CRYPTO=y CONFIG_DAX=y +CONFIG_ELF_CORE=y CONFIG_EROFS_FS=y -# CONFIG_ETHERNET is not set CONFIG_EXT4_FS=m +CONFIG_EXTCON=y CONFIG_FILE_LOCKING=y +CONFIG_INPUT=y CONFIG_JFFS2_FS=y CONFIG_KERNEL_GZIP=y # CONFIG_KERNEL_XZ is not set CONFIG_LIBCRC32C=y -CONFIG_MAILBOX=y -# CONFIG_MDIO_DEVICE is not set CONFIG_MMC=y CONFIG_MSDOS_FS=m CONFIG_MSDOS_PARTITION=y CONFIG_MTD_BLOCK=m CONFIG_NLS_CODEPAGE_936=m -# CONFIG_PHYLIB is not set CONFIG_PHY_ROCKCHIP_CSI2_DPHY=y +CONFIG_RK_DMABUF_PROCFS=y CONFIG_ROCKCHIP_DVBM=y CONFIG_ROCKCHIP_HW_DECOMPRESS=y CONFIG_ROCKCHIP_MULTI_RGA=y +CONFIG_ROCKCHIP_OPP=y CONFIG_ROCKCHIP_RAMDISK=y CONFIG_ROCKCHIP_RGA_PROC_FS=y CONFIG_ROCKCHIP_THUNDER_BOOT=y CONFIG_ROCKCHIP_THUNDER_BOOT_DEFER_FREE_MEMBLOCK=y CONFIG_ROCKCHIP_VENDOR_STORAGE=m # CONFIG_SLUB_SYSFS is not set -CONFIG_SND_SIMPLE_CARD=m -CONFIG_SND_SIMPLE_CARD_UTILS=m +# CONFIG_SND_SIMPLE_CARD is not set +CONFIG_SND_SOC_DUMMY_CODEC=m +CONFIG_SND_SOC_RK_DSM=m CONFIG_SND_SOC_ROCKCHIP=m +CONFIG_SND_SOC_ROCKCHIP_ASRC=m +CONFIG_SND_SOC_ROCKCHIP_PDM_V2=m CONFIG_SND_SOC_ROCKCHIP_SAI=m -CONFIG_SND_SOC_RV1106=m CONFIG_SPI=y +CONFIG_SQUASHFS_LZ4=y CONFIG_VFAT_FS=m +# CONFIG_VIDEO_RK_IRCUT is not set +CONFIG_VIDEO_ROCKCHIP_AIISP=y +CONFIG_VIDEO_ROCKCHIP_AVSP=y CONFIG_VIDEO_ROCKCHIP_CIF=y +CONFIG_VIDEO_ROCKCHIP_FEC=y CONFIG_VIDEO_ROCKCHIP_ISP=y CONFIG_VIDEO_ROCKCHIP_VPSS=y CONFIG_VIDEO_SC200AI=y CONFIG_VIDEO_SC450AI=y +CONFIG_VIDEO_SC850SL=y # CONFIG_AD2S1200 is not set # CONFIG_AD2S1210 is not set # CONFIG_AD2S90 is not set @@ -109,16 +118,14 @@ CONFIG_VIDEO_SC450AI=y # CONFIG_ADXRS290 is not set # CONFIG_ADXRS450 is not set # CONFIG_AFE4403 is not set -# CONFIG_ALTERA_MBOX is not set -# CONFIG_ARM_MHU is not set -# CONFIG_ARM_MHU_V2 is not set -# CONFIG_ARM_SCPI_PROTOCOL is not set # CONFIG_AS3935 is not set # CONFIG_BMA220 is not set # CONFIG_BMC150_MAGN_SPI is not set # CONFIG_BMI088_ACCEL is not set # CONFIG_BMI160_SPI is not set # CONFIG_BSD_DISKLABEL is not set +# CONFIG_CHARGER_BQ24190 is not set +CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y # CONFIG_CRYPTO_842 is not set # CONFIG_CRYPTO_ADIANTUM is not set # CONFIG_CRYPTO_AEGIS128 is not set @@ -215,8 +222,11 @@ CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y # CONFIG_CRYPTO_XTS is not set # CONFIG_CRYPTO_XXHASH is not set # CONFIG_CRYPTO_ZSTD is not set +# CONFIG_DM9051 is not set # CONFIG_EEPROM_93XX46 is not set # CONFIG_EEPROM_AT25 is not set +# CONFIG_ENC28J60 is not set +# CONFIG_ENCX24J600 is not set # CONFIG_EROFS_FS_DEBUG is not set # CONFIG_EROFS_FS_XATTR is not set # CONFIG_EROFS_FS_ZIP is not set @@ -224,6 +234,14 @@ CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y # CONFIG_EXT4_FS_POSIX_ACL is not set # CONFIG_EXT4_FS_SECURITY is not set CONFIG_EXT4_USE_FOR_EXT2=y +# CONFIG_EXTCON_ADC_JACK is not set +# CONFIG_EXTCON_FSA9480 is not set +# CONFIG_EXTCON_GPIO is not set +# CONFIG_EXTCON_MAX3355 is not set +# CONFIG_EXTCON_PTN5150 is not set +# CONFIG_EXTCON_RT8973A is not set +# CONFIG_EXTCON_SM5502 is not set +# CONFIG_EXTCON_USB_GPIO is not set # CONFIG_EZX_PCAP is not set CONFIG_FAT_DEFAULT_CODEPAGE=936 CONFIG_FAT_DEFAULT_IOCHARSET="cp936" @@ -241,10 +259,27 @@ CONFIG_FS_MBCACHE=m # CONFIG_GPIO_PISOSR is not set # CONFIG_GPIO_XRA1403 is not set # CONFIG_HI8435 is not set +# CONFIG_HID is not set +# CONFIG_I2C_HID_OF is not set +# CONFIG_I2C_HID_OF_ELAN is not set +# CONFIG_I2C_HID_OF_GOODIX is not set # CONFIG_IIO_SSP_SENSORHUB is not set CONFIG_INITCALL_ASYNC=y # CONFIG_INITRAMFS_FORCE is not set CONFIG_INITRAMFS_SOURCE="" +# CONFIG_INPUT_EVBUG is not set +CONFIG_INPUT_EVDEV=y +# CONFIG_INPUT_FF_MEMLESS is not set +# CONFIG_INPUT_JOYDEV is not set +# CONFIG_INPUT_JOYSTICK is not set +CONFIG_INPUT_KEYBOARD=y +# CONFIG_INPUT_MATRIXKMAP is not set +# CONFIG_INPUT_MISC is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_INPUT_MOUSEDEV is not set +# CONFIG_INPUT_SPARSEKMAP is not set +# CONFIG_INPUT_TABLET is not set +# CONFIG_INPUT_TOUCHSCREEN is not set # CONFIG_INV_ICM42600_SPI is not set # CONFIG_INV_ICM42670_SPI is not set # CONFIG_INV_MPU6050_SPI is not set @@ -264,6 +299,36 @@ CONFIG_JFFS2_FS_WRITEBUFFER=y # CONFIG_JFFS2_RUBIN is not set # CONFIG_JFFS2_SUMMARY is not set CONFIG_JFFS2_ZLIB=y +CONFIG_KEYBOARD_ADC=y +# CONFIG_KEYBOARD_ADP5588 is not set +# CONFIG_KEYBOARD_ADP5589 is not set +# CONFIG_KEYBOARD_ATKBD is not set +# CONFIG_KEYBOARD_BCM is not set +# CONFIG_KEYBOARD_CAP11XX is not set +# CONFIG_KEYBOARD_CYPRESS_SF is not set +# CONFIG_KEYBOARD_DLINK_DIR685 is not set +# CONFIG_KEYBOARD_GPIO is not set +# CONFIG_KEYBOARD_GPIO_POLLED is not set +# CONFIG_KEYBOARD_LKKBD is not set +# CONFIG_KEYBOARD_LM8333 is not set +# CONFIG_KEYBOARD_MATRIX is not set +# CONFIG_KEYBOARD_MAX7359 is not set +# CONFIG_KEYBOARD_MCS is not set +# CONFIG_KEYBOARD_MPR121 is not set +# CONFIG_KEYBOARD_NEWTON is not set +# CONFIG_KEYBOARD_OMAP4 is not set +# CONFIG_KEYBOARD_OPENCORES is not set +# CONFIG_KEYBOARD_PINEPHONE is not set +# CONFIG_KEYBOARD_QT1050 is not set +# CONFIG_KEYBOARD_QT1070 is not set +# CONFIG_KEYBOARD_QT2160 is not set +# CONFIG_KEYBOARD_SAMSUNG is not set +# CONFIG_KEYBOARD_STOWAWAY is not set +# CONFIG_KEYBOARD_SUNKBD is not set +# CONFIG_KEYBOARD_TCA6416 is not set +# CONFIG_KEYBOARD_TCA8418 is not set +# CONFIG_KEYBOARD_XTKBD is not set +# CONFIG_KS8851 is not set # CONFIG_LATTICE_ECP3_CONFIG is not set # CONFIG_LMK04832 is not set # CONFIG_LTC1660 is not set @@ -271,8 +336,7 @@ CONFIG_JFFS2_ZLIB=y # CONFIG_LTC2632 is not set # CONFIG_LTC2688 is not set # CONFIG_LTC2983 is not set -# CONFIG_MAILBOX_POLL_PERIOD_US is not set -# CONFIG_MAILBOX_TEST is not set +CONFIG_LZ4_DECOMPRESS=y # CONFIG_MAX1027 is not set # CONFIG_MAX11100 is not set # CONFIG_MAX1118 is not set @@ -322,6 +386,7 @@ CONFIG_MMC_QUEUE_DEPTH=1 # CONFIG_MMC_USDHI6ROL0 is not set # CONFIG_MOXTET is not set # CONFIG_MPL115_SPI is not set +# CONFIG_MSE102X is not set CONFIG_MTD_BLKDEVS=m # CONFIG_MTD_DATAFLASH is not set # CONFIG_MTD_MCHP23K256 is not set @@ -357,11 +422,12 @@ CONFIG_MTD_SPI_NOR_WINBOND=y # CONFIG_MTD_SPI_NOR_XMC is not set # CONFIG_MTD_SPI_NOR_XTX is not set # CONFIG_MTD_SST25L is not set +CONFIG_NET_VENDOR_ADI=y # CONFIG_PI433 is not set -# CONFIG_PL320_MBOX is not set -# CONFIG_PLATFORM_MHU is not set # CONFIG_PWRSEQ_EMMC is not set CONFIG_PWRSEQ_SIMPLE=y +# CONFIG_QCA7000_SPI is not set +# CONFIG_RC_CORE is not set # CONFIG_RD_BZIP2 is not set # CONFIG_RD_GZIP is not set # CONFIG_RD_LZ4 is not set @@ -371,16 +437,15 @@ CONFIG_PWRSEQ_SIMPLE=y # CONFIG_RD_ZSTD is not set CONFIG_REGMAP_SPI=y # CONFIG_REGULATOR_TPS6524X is not set +# CONFIG_RMI4_CORE is not set # CONFIG_ROCKCHIP_HW_DECOMPRESS_TEST is not set -# CONFIG_ROCKCHIP_MBOX is not set # CONFIG_ROCKCHIP_MMC_VENDOR_STORAGE is not set CONFIG_ROCKCHIP_MTD_VENDOR_STORAGE=m # CONFIG_ROCKCHIP_RAM_VENDOR_STORAGE is not set +# CONFIG_ROCKCHIP_REMOTECTL is not set CONFIG_ROCKCHIP_RGA_DEBUGGER=y CONFIG_ROCKCHIP_THUNDER_BOOT_MMC=y -# CONFIG_ROCKCHIP_THUNDER_BOOT_SERVICE is not set CONFIG_ROCKCHIP_THUNDER_BOOT_SFC=y -# CONFIG_RPMSG_QCOM_GLINK_RPM is not set # CONFIG_RTC_DRV_DS1302 is not set # CONFIG_RTC_DRV_DS1305 is not set # CONFIG_RTC_DRV_DS1343 is not set @@ -399,15 +464,21 @@ CONFIG_ROCKCHIP_THUNDER_BOOT_SFC=y # CONFIG_SCA3300 is not set # CONFIG_SDIO_UART is not set # CONFIG_SENSORS_HMC5843_SPI is not set +# CONFIG_SENSORS_LIS3_I2C is not set +# CONFIG_SENSORS_LIS3_SPI is not set # CONFIG_SENSORS_RM3100_SPI is not set +# CONFIG_SENSOR_DEVICE is not set # CONFIG_SERIAL_MAX3100 is not set # CONFIG_SERIAL_MAX310X is not set +CONFIG_SND_JACK_INPUT_DEV=y # CONFIG_SND_SOC_ADAU1372_SPI is not set # CONFIG_SND_SOC_ADAU1761_SPI is not set # CONFIG_SND_SOC_AK4104 is not set # CONFIG_SND_SOC_CS35L41_SPI is not set # CONFIG_SND_SOC_CS35L45_SPI is not set # CONFIG_SND_SOC_CS4271_SPI is not set +# CONFIG_SND_SOC_CS42L52 is not set +# CONFIG_SND_SOC_CS42L56 is not set # CONFIG_SND_SOC_ES8328_SPI is not set # CONFIG_SND_SOC_PCM179X_SPI is not set # CONFIG_SND_SOC_PCM186X_SPI is not set @@ -415,6 +486,7 @@ CONFIG_ROCKCHIP_THUNDER_BOOT_SFC=y # CONFIG_SND_SOC_PCM3168A_SPI is not set # CONFIG_SND_SOC_PCM512x_SPI is not set # CONFIG_SND_SOC_RK3399_GRU_SOUND is not set +CONFIG_SND_SOC_ROCKCHIP_MULTICODECS=m # CONFIG_SND_SOC_ROCKCHIP_SPI_CODEC is not set # CONFIG_SND_SOC_SSM2602_SPI is not set # CONFIG_SND_SOC_TLV320AIC23_SPI is not set @@ -423,11 +495,13 @@ CONFIG_ROCKCHIP_THUNDER_BOOT_SFC=y # CONFIG_SND_SOC_WM8731_SPI is not set # CONFIG_SND_SOC_WM8770 is not set # CONFIG_SND_SOC_WM8804_SPI is not set +# CONFIG_SND_SOC_WM8962 is not set # CONFIG_SND_SOC_ZL38060 is not set # CONFIG_SND_SPI is not set # CONFIG_SOLARIS_X86_PARTITION is not set # CONFIG_SPI_ALTERA is not set # CONFIG_SPI_AMD is not set +# CONFIG_SPI_AX88796C is not set # CONFIG_SPI_AXI_SPI_ENGINE is not set # CONFIG_SPI_BITBANG is not set # CONFIG_SPI_CADENCE is not set From 80623b62c5c06bcf7900c5c681afd1ea92ee590f Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Fri, 13 Jun 2025 17:15:46 +0800 Subject: [PATCH 23/49] ARM: configs: Update rv1126b-rndis Signed-off-by: Weiwen Chen Change-Id: Ib965502a423e0370f56184a48a8dcd1313f77dce --- arch/arm/configs/rv1126b-rndis.config | 39 ++++++++++++++++++--------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/arch/arm/configs/rv1126b-rndis.config b/arch/arm/configs/rv1126b-rndis.config index b3d39b93ea11..54cdea391359 100644 --- a/arch/arm/configs/rv1126b-rndis.config +++ b/arch/arm/configs/rv1126b-rndis.config @@ -1,5 +1,6 @@ -CONFIG_CONFIGFS_FS=y +CONFIG_CONFIGFS_FS=m CONFIG_EXTCON=y +CONFIG_PHY_ROCKCHIP_NANENG_COMBO_PHY=m CONFIG_USB_SUPPORT=y CONFIG_VIDEOBUF2_DMA_SG=m # CONFIG_APPLE_MFI_FASTCHARGE is not set @@ -12,6 +13,7 @@ CONFIG_VIDEOBUF2_DMA_SG=m # CONFIG_EXTCON_RT8973A is not set # CONFIG_EXTCON_SM5502 is not set # CONFIG_EXTCON_USB_GPIO is not set +# CONFIG_HISI_HIKEY_USB is not set # CONFIG_I2C_CP2615 is not set # CONFIG_I2C_DIOLAN_U2C is not set # CONFIG_I2C_ROBOTFUZZ_OSIF is not set @@ -55,18 +57,18 @@ CONFIG_USB_CONFIGFS=m # CONFIG_USB_CONFIGFS_ECM_SUBSET is not set # CONFIG_USB_CONFIGFS_EEM is not set CONFIG_USB_CONFIGFS_F_FS=y -# CONFIG_USB_CONFIGFS_F_HID is not set +CONFIG_USB_CONFIGFS_F_HID=y # CONFIG_USB_CONFIGFS_F_LB_SS is not set # CONFIG_USB_CONFIGFS_F_MIDI is not set # CONFIG_USB_CONFIGFS_F_PRINTER is not set -# CONFIG_USB_CONFIGFS_F_UAC1 is not set +CONFIG_USB_CONFIGFS_F_UAC1=y # CONFIG_USB_CONFIGFS_F_UAC1_LEGACY is not set -# CONFIG_USB_CONFIGFS_F_UAC2 is not set +CONFIG_USB_CONFIGFS_F_UAC2=y CONFIG_USB_CONFIGFS_F_UVC=y # CONFIG_USB_CONFIGFS_MASS_STORAGE is not set # CONFIG_USB_CONFIGFS_NCM is not set # CONFIG_USB_CONFIGFS_OBEX is not set -CONFIG_USB_CONFIGFS_RNDIS=y +# CONFIG_USB_CONFIGFS_RNDIS is not set # CONFIG_USB_CONFIGFS_SERIAL is not set CONFIG_USB_CONFIGFS_UEVENT=y # CONFIG_USB_CONN_GPIO is not set @@ -76,12 +78,16 @@ CONFIG_USB_DEFAULT_PERSIST=y # CONFIG_USB_DUMMY_HCD is not set # CONFIG_USB_DWC2 is not set CONFIG_USB_DWC3=m -# CONFIG_USB_DWC3_DUAL_ROLE is not set -CONFIG_USB_DWC3_GADGET=y +CONFIG_USB_DWC3_DUAL_ROLE=y +# CONFIG_USB_DWC3_GADGET is not set # CONFIG_USB_DWC3_HOST is not set CONFIG_USB_DWC3_OF_SIMPLE=m # CONFIG_USB_DYNAMIC_MINORS is not set -# CONFIG_USB_EHCI_HCD is not set +# CONFIG_USB_EHCI_FSL is not set +CONFIG_USB_EHCI_HCD=m +CONFIG_USB_EHCI_HCD_PLATFORM=m +# CONFIG_USB_EHCI_ROOT_HUB_TT is not set +CONFIG_USB_EHCI_TT_NEWSCHED=y # CONFIG_USB_EHSET_TEST_FIXTURE is not set # CONFIG_USB_EMI26 is not set # CONFIG_USB_EMI62 is not set @@ -94,7 +100,9 @@ CONFIG_USB_DWC3_OF_SIMPLE=m # CONFIG_USB_FUNCTIONFS is not set # CONFIG_USB_FUSB300 is not set CONFIG_USB_F_FS=m -CONFIG_USB_F_RNDIS=m +CONFIG_USB_F_HID=m +CONFIG_USB_F_UAC1=m +CONFIG_USB_F_UAC2=m CONFIG_USB_F_UVC=m CONFIG_USB_GADGET=m # CONFIG_USB_GADGETFS is not set @@ -138,7 +146,8 @@ CONFIG_USB_LIBCOMPOSITE=m # CONFIG_USB_MV_UDC is not set # CONFIG_USB_NET2272 is not set # CONFIG_USB_NET_DRIVERS is not set -# CONFIG_USB_OHCI_HCD is not set +CONFIG_USB_OHCI_HCD=m +CONFIG_USB_OHCI_HCD_PLATFORM=m # CONFIG_USB_ONBOARD_HUB is not set # CONFIG_USB_OTG is not set # CONFIG_USB_OTG_DISABLE_EXTERNAL_HUB is not set @@ -149,9 +158,10 @@ CONFIG_USB_LIBCOMPOSITE=m # CONFIG_USB_R8A66597 is not set # CONFIG_USB_R8A66597_HCD is not set # CONFIG_USB_RAW_GADGET is not set -# CONFIG_USB_ROLE_SWITCH is not set +CONFIG_USB_ROLE_SWITCH=m # CONFIG_USB_SERIAL is not set # CONFIG_USB_SEVSEG is not set +# CONFIG_USB_SISUSBVGA is not set # CONFIG_USB_SL811_HCD is not set # CONFIG_USB_SNP_UDC_PLAT is not set # CONFIG_USB_TEST is not set @@ -159,9 +169,12 @@ CONFIG_USB_LIBCOMPOSITE=m # CONFIG_USB_TRANCEVIBRATOR is not set # CONFIG_USB_ULPI is not set # CONFIG_USB_ULPI_BUS is not set -CONFIG_USB_U_ETHER=m +CONFIG_USB_U_AUDIO=m # CONFIG_USB_WDM is not set -# CONFIG_USB_XHCI_HCD is not set +# CONFIG_USB_XHCI_DBGCAP is not set +CONFIG_USB_XHCI_HCD=m +# CONFIG_USB_XHCI_PCI_RENESAS is not set +CONFIG_USB_XHCI_PLATFORM=m # CONFIG_USB_YUREX is not set # CONFIG_USB_ZERO is not set # CONFIG_XILLYUSB is not set From 19816b4be81b4d6adef488783e9aa251208747f5 Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Fri, 13 Jun 2025 17:49:56 +0800 Subject: [PATCH 24/49] ARM: configs: Update rv1126b-nand Signed-off-by: Weiwen Chen Change-Id: I41251faab46d94508e54cfaac07b75bec994cd4d --- arch/arm/configs/rv1126b-nand.config | 268 +++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) diff --git a/arch/arm/configs/rv1126b-nand.config b/arch/arm/configs/rv1126b-nand.config index c356b4f35bfd..9f4c2f0163c5 100644 --- a/arch/arm/configs/rv1126b-nand.config +++ b/arch/arm/configs/rv1126b-nand.config @@ -1,6 +1,81 @@ CONFIG_CRC16=y CONFIG_CRYPTO=y CONFIG_MTD_UBI=y +CONFIG_SPI=y +# CONFIG_AD2S1200 is not set +# CONFIG_AD2S1210 is not set +# CONFIG_AD2S90 is not set +# CONFIG_AD3552R is not set +# CONFIG_AD5360 is not set +# CONFIG_AD5421 is not set +# CONFIG_AD5449 is not set +# CONFIG_AD5504 is not set +# CONFIG_AD5592R is not set +# CONFIG_AD5624R_SPI is not set +# CONFIG_AD5686_SPI is not set +# CONFIG_AD5755 is not set +# CONFIG_AD5758 is not set +# CONFIG_AD5761 is not set +# CONFIG_AD5764 is not set +# CONFIG_AD5766 is not set +# CONFIG_AD5770R is not set +# CONFIG_AD5791 is not set +# CONFIG_AD7124 is not set +# CONFIG_AD7192 is not set +# CONFIG_AD7266 is not set +# CONFIG_AD7280 is not set +# CONFIG_AD7292 is not set +# CONFIG_AD7293 is not set +# CONFIG_AD7298 is not set +# CONFIG_AD7303 is not set +# CONFIG_AD74413R is not set +# CONFIG_AD7476 is not set +# CONFIG_AD7606_IFACE_SPI is not set +# CONFIG_AD7766 is not set +# CONFIG_AD7768_1 is not set +# CONFIG_AD7780 is not set +# CONFIG_AD7791 is not set +# CONFIG_AD7793 is not set +# CONFIG_AD7816 is not set +# CONFIG_AD7887 is not set +# CONFIG_AD7923 is not set +# CONFIG_AD7949 is not set +# CONFIG_AD8366 is not set +# CONFIG_AD8801 is not set +# CONFIG_AD9523 is not set +# CONFIG_AD9832 is not set +# CONFIG_AD9834 is not set +# CONFIG_ADA4250 is not set +# CONFIG_ADF4350 is not set +# CONFIG_ADF4371 is not set +# CONFIG_ADIS16080 is not set +# CONFIG_ADIS16130 is not set +# CONFIG_ADIS16136 is not set +# CONFIG_ADIS16201 is not set +# CONFIG_ADIS16203 is not set +# CONFIG_ADIS16209 is not set +# CONFIG_ADIS16240 is not set +# CONFIG_ADIS16260 is not set +# CONFIG_ADIS16400 is not set +# CONFIG_ADIS16460 is not set +# CONFIG_ADIS16475 is not set +# CONFIG_ADIS16480 is not set +# CONFIG_ADMV1013 is not set +# CONFIG_ADMV4420 is not set +# CONFIG_ADRF6780 is not set +# CONFIG_ADXL313_SPI is not set +# CONFIG_ADXL345_SPI is not set +# CONFIG_ADXL355_SPI is not set +# CONFIG_ADXL367_SPI is not set +# CONFIG_ADXL372_SPI is not set +# CONFIG_ADXRS290 is not set +# CONFIG_ADXRS450 is not set +# CONFIG_AFE4403 is not set +# CONFIG_AS3935 is not set +# CONFIG_BMA220 is not set +# CONFIG_BMC150_MAGN_SPI is not set +# CONFIG_BMI088_ACCEL is not set +# CONFIG_BMI160_SPI is not set # CONFIG_CRYPTO_842 is not set CONFIG_CRYPTO_ACOMP2=y # CONFIG_CRYPTO_ADIANTUM is not set @@ -103,14 +178,204 @@ CONFIG_CRYPTO_MANAGER_DISABLE_TESTS=y # CONFIG_CRYPTO_XTS is not set # CONFIG_CRYPTO_XXHASH is not set # CONFIG_CRYPTO_ZSTD is not set +# CONFIG_DM9051 is not set +# CONFIG_EEPROM_93XX46 is not set +# CONFIG_EEPROM_AT25 is not set +# CONFIG_ENC28J60 is not set +# CONFIG_ENCX24J600 is not set +# CONFIG_EZX_PCAP is not set +# CONFIG_FXLS8962AF_SPI is not set +# CONFIG_FXOS8700_SPI is not set +# CONFIG_GPIO_74X164 is not set +# CONFIG_GPIO_MAX3191X is not set +# CONFIG_GPIO_MAX7301 is not set +# CONFIG_GPIO_MC33880 is not set +# CONFIG_GPIO_PISOSR is not set +# CONFIG_GPIO_XRA1403 is not set +# CONFIG_HI8435 is not set +# CONFIG_IIO_SSP_SENSORHUB is not set +# CONFIG_INV_ICM42600_SPI is not set +# CONFIG_INV_ICM42670_SPI is not set +# CONFIG_INV_MPU6050_SPI is not set +# CONFIG_KS8851 is not set +# CONFIG_LATTICE_ECP3_CONFIG is not set +# CONFIG_LMK04832 is not set +# CONFIG_LTC1660 is not set +# CONFIG_LTC2496 is not set +# CONFIG_LTC2632 is not set +# CONFIG_LTC2688 is not set +# CONFIG_LTC2983 is not set CONFIG_LZO_COMPRESS=y CONFIG_LZO_DECOMPRESS=y +# CONFIG_MAX1027 is not set +# CONFIG_MAX11100 is not set +# CONFIG_MAX1118 is not set +# CONFIG_MAX11205 is not set +# CONFIG_MAX1241 is not set +# CONFIG_MAX31856 is not set +# CONFIG_MAX31865 is not set +# CONFIG_MAX5481 is not set +# CONFIG_MAX5487 is not set +# CONFIG_MAXIM_THERMOCOUPLE is not set +# CONFIG_MCP320X is not set +# CONFIG_MCP3911 is not set +# CONFIG_MCP41010 is not set +# CONFIG_MCP4131 is not set +# CONFIG_MCP4922 is not set +# CONFIG_MFD_ARIZONA_SPI is not set +# CONFIG_MFD_CPCAP is not set +# CONFIG_MFD_DA9052_SPI is not set +# CONFIG_MFD_INTEL_M10_BMC is not set +# CONFIG_MFD_MC13XXX_SPI is not set +# CONFIG_MFD_OCELOT is not set +# CONFIG_MFD_RK806_SPI is not set +# CONFIG_MFD_RSMU_SPI is not set +# CONFIG_MFD_TPS65912_SPI is not set +# CONFIG_MFD_WM831X_SPI is not set +# CONFIG_MICREL_KS8995MA is not set +# CONFIG_MMA7455_SPI is not set +# CONFIG_MOXTET is not set +# CONFIG_MPL115_SPI is not set +# CONFIG_MSE102X is not set +# CONFIG_MTD_DATAFLASH is not set +# CONFIG_MTD_MCHP23K256 is not set +# CONFIG_MTD_MCHP48L640 is not set +CONFIG_MTD_NAND_BBT_USING_FLASH=y +CONFIG_MTD_NAND_CORE=y +CONFIG_MTD_NAND_ECC=y +CONFIG_MTD_SPI_NAND=y +CONFIG_MTD_SPI_NAND_ATO=y +CONFIG_MTD_SPI_NAND_BIWIN=y +CONFIG_MTD_SPI_NAND_DEVICE_AUTOSELECT=y +CONFIG_MTD_SPI_NAND_DOSILICON=y +CONFIG_MTD_SPI_NAND_ESMT=y +CONFIG_MTD_SPI_NAND_ETRON=y +CONFIG_MTD_SPI_NAND_FMSH=y +CONFIG_MTD_SPI_NAND_FORESEE=y +CONFIG_MTD_SPI_NAND_GIGADEVICE=y +CONFIG_MTD_SPI_NAND_GSTO=y +CONFIG_MTD_SPI_NAND_HIKSEMI=y +CONFIG_MTD_SPI_NAND_HYF=y +CONFIG_MTD_SPI_NAND_JSC=y +CONFIG_MTD_SPI_NAND_MACRONIX=y +CONFIG_MTD_SPI_NAND_MICRON=y +CONFIG_MTD_SPI_NAND_PARAGON=y +CONFIG_MTD_SPI_NAND_SILICONGO=y +CONFIG_MTD_SPI_NAND_SKYHIGH=y +CONFIG_MTD_SPI_NAND_TOSHIBA=y +CONFIG_MTD_SPI_NAND_UNIM=y +CONFIG_MTD_SPI_NAND_WINBOND=y +CONFIG_MTD_SPI_NAND_XINCUN=y +CONFIG_MTD_SPI_NAND_XTX=y +CONFIG_MTD_SPI_NAND_ZBIT=y +# CONFIG_MTD_SPI_NOR is not set +# CONFIG_MTD_SST25L is not set CONFIG_MTD_UBI_BEB_LIMIT=20 CONFIG_MTD_UBI_BLOCK=y # CONFIG_MTD_UBI_FASTMAP is not set # CONFIG_MTD_UBI_GLUEBI is not set CONFIG_MTD_UBI_WL_THRESHOLD=4096 +# CONFIG_NET_VENDOR_ADI is not set +# CONFIG_PI433 is not set +# CONFIG_QCA7000_SPI is not set +CONFIG_REGMAP_SPI=y +# CONFIG_REGULATOR_TPS6524X is not set +# CONFIG_ROCKCHIP_MTD_VENDOR_STORAGE is not set +# CONFIG_RTC_DRV_DS1302 is not set +# CONFIG_RTC_DRV_DS1305 is not set +# CONFIG_RTC_DRV_DS1343 is not set +# CONFIG_RTC_DRV_DS1347 is not set +# CONFIG_RTC_DRV_DS1390 is not set +# CONFIG_RTC_DRV_M41T93 is not set +# CONFIG_RTC_DRV_M41T94 is not set +# CONFIG_RTC_DRV_MAX6902 is not set +# CONFIG_RTC_DRV_MAX6916 is not set +# CONFIG_RTC_DRV_MCP795 is not set +# CONFIG_RTC_DRV_PCF2123 is not set +# CONFIG_RTC_DRV_R9701 is not set +# CONFIG_RTC_DRV_RS5C348 is not set +# CONFIG_RTC_DRV_RX4581 is not set +# CONFIG_SCA3000 is not set +# CONFIG_SCA3300 is not set +# CONFIG_SENSORS_HMC5843_SPI is not set +# CONFIG_SENSORS_RM3100_SPI is not set +# CONFIG_SERIAL_MAX3100 is not set +# CONFIG_SERIAL_MAX310X is not set CONFIG_SGL_ALLOC=y +# CONFIG_SND_SOC_ADAU1372_SPI is not set +# CONFIG_SND_SOC_ADAU1761_SPI is not set +# CONFIG_SND_SOC_AK4104 is not set +# CONFIG_SND_SOC_CS35L41_SPI is not set +# CONFIG_SND_SOC_CS35L45_SPI is not set +# CONFIG_SND_SOC_CS4271_SPI is not set +# CONFIG_SND_SOC_ES8328_SPI is not set +# CONFIG_SND_SOC_PCM179X_SPI is not set +# CONFIG_SND_SOC_PCM186X_SPI is not set +# CONFIG_SND_SOC_PCM3060_SPI is not set +# CONFIG_SND_SOC_PCM3168A_SPI is not set +# CONFIG_SND_SOC_PCM512x_SPI is not set +# CONFIG_SND_SOC_RK3399_GRU_SOUND is not set +# CONFIG_SND_SOC_ROCKCHIP_SPI_CODEC is not set +# CONFIG_SND_SOC_SSM2602_SPI is not set +# CONFIG_SND_SOC_TLV320AIC23_SPI is not set +# CONFIG_SND_SOC_TLV320AIC32X4_SPI is not set +# CONFIG_SND_SOC_TLV320AIC3X_SPI is not set +# CONFIG_SND_SOC_WM8731_SPI is not set +# CONFIG_SND_SOC_WM8770 is not set +# CONFIG_SND_SOC_WM8804_SPI is not set +# CONFIG_SND_SOC_ZL38060 is not set +# CONFIG_SND_SPI is not set +# CONFIG_SPI_ALTERA is not set +# CONFIG_SPI_AMD is not set +# CONFIG_SPI_AX88796C is not set +# CONFIG_SPI_AXI_SPI_ENGINE is not set +# CONFIG_SPI_BITBANG is not set +# CONFIG_SPI_CADENCE is not set +# CONFIG_SPI_CADENCE_QUADSPI is not set +# CONFIG_SPI_CADENCE_XSPI is not set +# CONFIG_SPI_DEBUG is not set +# CONFIG_SPI_DESIGNWARE is not set +# CONFIG_SPI_FSL_SPI is not set +# CONFIG_SPI_GPIO is not set +# CONFIG_SPI_LOOPBACK_TEST is not set +CONFIG_SPI_MASTER=y +CONFIG_SPI_MEM=y +# CONFIG_SPI_MICROCHIP_CORE is not set +# CONFIG_SPI_MICROCHIP_CORE_QSPI is not set +# CONFIG_SPI_MUX is not set +# CONFIG_SPI_MXIC is not set +# CONFIG_SPI_NXP_FLEXSPI is not set +# CONFIG_SPI_OC_TINY is not set +# CONFIG_SPI_PL022 is not set +# CONFIG_SPI_ROCKCHIP is not set +# CONFIG_SPI_ROCKCHIP_FLEXBUS_FSPI is not set +# CONFIG_SPI_ROCKCHIP_FLEXBUS_SPI is not set +CONFIG_SPI_ROCKCHIP_SFC=y +# CONFIG_SPI_ROCKCHIP_SLAVE is not set +# CONFIG_SPI_SC18IS602 is not set +# CONFIG_SPI_SIFIVE is not set +# CONFIG_SPI_SLAVE is not set +# CONFIG_SPI_SPIDEV is not set +# CONFIG_SPI_TLE62X0 is not set +# CONFIG_SPI_XCOMM is not set +# CONFIG_SPI_XILINX is not set +# CONFIG_SPI_ZYNQMP_GQSPI is not set +# CONFIG_TI_ADC0832 is not set +# CONFIG_TI_ADC084S021 is not set +# CONFIG_TI_ADC108S102 is not set +# CONFIG_TI_ADC12138 is not set +# CONFIG_TI_ADC128S052 is not set +# CONFIG_TI_ADC161S626 is not set +# CONFIG_TI_ADS124S08 is not set +# CONFIG_TI_ADS131E08 is not set +# CONFIG_TI_ADS7950 is not set +# CONFIG_TI_ADS8344 is not set +# CONFIG_TI_ADS8688 is not set +# CONFIG_TI_DAC082S085 is not set +# CONFIG_TI_DAC7311 is not set +# CONFIG_TI_DAC7612 is not set +# CONFIG_TI_TLC4541 is not set +# CONFIG_TI_TSC2046 is not set # CONFIG_UBIFS_ATIME_SUPPORT is not set CONFIG_UBIFS_FS=y CONFIG_UBIFS_FS_ADVANCED_COMPR=y @@ -120,6 +385,9 @@ CONFIG_UBIFS_FS_SECURITY=y CONFIG_UBIFS_FS_XATTR=y CONFIG_UBIFS_FS_ZLIB=y # CONFIG_UBIFS_FS_ZSTD is not set +# CONFIG_VIDEO_GS1662 is not set +# CONFIG_VIDEO_ROCKCHIP_PREISP is not set +# CONFIG_VIDEO_S5C73M3 is not set CONFIG_ZLIB_DEFLATE=y CONFIG_ZLIB_INFLATE=y # CONFIG_ZRAM is not set From ed3dda1f9b77dd6552caacd828391905699bf446 Mon Sep 17 00:00:00 2001 From: Luo Wei Date: Tue, 10 Jun 2025 18:33:03 +0800 Subject: [PATCH 25/49] arm64: dts: rockchip: rk3576-vehicle-evb: add v21 dts Change-Id: Ifd8ce1732210a313e9e662bdd3b089228f076d97 Signed-off-by: Luo Wei Signed-off-by: Cai Wenzhong --- arch/arm64/boot/dts/rockchip/Makefile | 1 + .../dts/rockchip/rk3576-vehicle-evb-v21.dts | 576 ++++++++++++++++++ 2 files changed, 577 insertions(+) create mode 100644 arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21.dts diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile index e5d0402f772f..72a54df04cfe 100644 --- a/arch/arm64/boot/dts/rockchip/Makefile +++ b/arch/arm64/boot/dts/rockchip/Makefile @@ -299,6 +299,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v20.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v20-amp.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v20-linux.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v20-ufs.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v21.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576s-evb1-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576s-evb1-v10-linux.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576s-tablet-v10.dtb diff --git a/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21.dts b/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21.dts new file mode 100644 index 000000000000..9e119c0a1b16 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21.dts @@ -0,0 +1,576 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include "rk3576.dtsi" +#include "rk3576-vehicle-evb-v20.dtsi" +#include "rk3576-vehicle-evb-v20-nca9539-io-expander.dtsi" +#include "rk3576-vehicle-evb-v20-serdes-mfd-display-maxim.dtsi" +#include "rk3576-vehicle-evb-v20-maxim-max96712-dphy0-ox03j10.dtsi" +#include "rk3576-vehicle-evb-v20-maxim-max96712-dphy3-sc233at.dtsi" +#include "rk3576-android.dtsi" + +/delete-node/ &vcc5v0_host_usb30; + +/ { + model = "Rockchip RK3576 VEHICLE EVB V21 Board"; + compatible = "rockchip,rk3576-vehicle-evb-v21", "rockchip,rk3576"; + + chosen: chosen { + bootargs = "earlycon=uart8250,mmio32,0x2ad40000 console=ttyFIQ0 rcupdate.rcu_expedited=1 rcu_nocbs=all spidev.bufsiz=131072"; + }; + + vehicle_dummy: vehicle-dummy { + status = "okay"; + compatible = "rockchip,vehicle-dummy-adc"; + io-channels = <&saradc 4>, <&saradc 5>, <&saradc 6>; + io-channel-names = "gear", "turn_left", "turn_right"; + }; + + vcc5v0_buck: vcc5v0-buck { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&gpio0 RK_PD3 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_buck_en>; + startup-delay-us = <2500>; + off-on-delay-us = <1500>; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + cluster_power_buck: cluster_power-buck { + compatible = "regulator-fixed"; + regulator-name = "cluster_power_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + //enable-active-high; + gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; + vin-supply = <&vcc_1v8_s0>; + pinctrl-names = "default"; + pinctrl-0 = <&cluster_buck_en>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + usb_otg_vcc5v_buck: usb_otg_vcc5v-buck { + compatible = "regulator-fixed"; + regulator-name = "usb_otg_vcc5v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + //enable-active-high; + gpio = <&i2c0_nca9539_gpio 0 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc5v0_buck>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + usb_host_vcc5v_buck: usb_host_vcc5v-buck { + compatible = "regulator-fixed"; + regulator-name = "usb_host_vcc5v_buck"; + //regulator-boot-on; + //regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 1 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc5v0_buck>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + lcd1_vcc12v_buck: lcd1_vcc12v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd1_vcc12v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 2 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + lcd2_vcc12v_buck: lcd2_vcc12v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd2_vcc12v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 3 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + lcd1_ser_vcc5v_buck: lcd1_ser_vcc5v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd1_ser_vcc5v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 4 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc5v0_buck>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + lcd2_ser_vcc5v_buck: lcd2_ser_vcc5v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd2_ser_vcc5v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 5 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc5v0_buck>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <5000000>; + }; + }; + + adsp_vcc12v_buck: adsp_vcc12v-buck { + compatible = "regulator-fixed"; + regulator-name = "adsp_vcc12v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c0_nca9539_gpio 6 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + lcd3_vcc12v_buck: lcd3_vcc12v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd3_vcc12v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 0 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + lcd3_vcc5v_buck: lcd3_vcc5v-buck { + compatible = "regulator-fixed"; + regulator-name = "lcd3_vcc5v_buck"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 1 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dcphy0_vcc12v_buck1: dcphy0_vcc12v-buck1 { + compatible = "regulator-fixed"; + regulator-name = "dcphy0_vcc12v_buck1"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 3 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dcphy0_vcc12v_buck2: dcphy0_vcc12v-buck2 { + compatible = "regulator-fixed"; + regulator-name = "dcphy0_vcc12v_buck2"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 4 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dcphy0_vcc12v_buck3: dcphy0_vcc12v-buck3 { + compatible = "regulator-fixed"; + regulator-name = "dcphy0_vcc12v_buck3"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 5 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dcphy0_vcc12v_buck4: dcphy0_vcc12v-buck4 { + compatible = "regulator-fixed"; + regulator-name = "dcphy0_vcc12v_buck4"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 6 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy0_vcc12v_buck1: dphy0_vcc12v-buck1 { + compatible = "regulator-fixed"; + regulator-name = "dphy0_vcc12v_buck1"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 7 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy0_vcc12v_buck2: dphy0_vcc12v-buck2 { + compatible = "regulator-fixed"; + regulator-name = "dphy0_vcc12v_buck2"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 8 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy0_vcc12v_buck3: dphy0_vcc12v-buck3 { + compatible = "regulator-fixed"; + regulator-name = "dphy0_vcc12v_buck3"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 9 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy0_vcc12v_buck4: dphy0_vcc12v-buck4 { + compatible = "regulator-fixed"; + regulator-name = "dphy0_vcc12v_buck4"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 10 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy3_vcc12v_buck1: dphy3_vcc12v-buck1 { + compatible = "regulator-fixed"; + regulator-name = "dphy3_vcc12v_buck1"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 11 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy3_vcc12v_buck2: dphy3_vcc12v-buck2 { + compatible = "regulator-fixed"; + regulator-name = "dphy3_vcc12v_buck2"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 12 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy3_vcc12v_buck3: dphy3_vcc12v-buck3 { + compatible = "regulator-fixed"; + regulator-name = "dphy3_vcc12v_buck3"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 13 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; + + dphy3_vcc12v_buck4: dphy3_vcc12v-buck4 { + compatible = "regulator-fixed"; + regulator-name = "dphy3_vcc12v_buck4"; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + enable-active-high; + gpio = <&i2c7_nca9539_gpio 14 GPIO_ACTIVE_HIGH>; + startup-delay-us = <2000>; + off-on-delay-us = <16000>; + vin-supply = <&vcc12v_dcin>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <12000000>; + }; + }; +}; + +&dfi { + status = "disabled"; +}; + +&dmc { + status = "disabled"; +}; + +&gmac1 { + status = "disabled"; +}; + +&hym8563 { + status = "disabled"; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2m0_xfer>; +}; + +/*edp*/ +&i2c3_max96752 { + use-reg-check-work; + vpower-supply = <&lcd1_vcc12v_buck>; +}; + +/*edp touch*/ +&i2c3_himax { + himax,irq-gpio = <&gpio0 RK_PB6 IRQ_TYPE_EDGE_FALLING>; +}; + +/*dp*/ +&i2c5_max96745 { + lock-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; +}; + +&i2c5_ilitek { + interrupt-parent = <&gpio4>; + interrupts = ; +}; + +/*dp*/ +&i2c5_max96752 { + use-reg-check-work; + vpower-supply = <&lcd2_vcc12v_buck>; +}; + +&i2c8_max96789 { + route-enable; +}; + +/*dsi*/ +&i2c8_max96752 { + use-reg-check-work; + vpower-supply = <&lcd3_vcc12v_buck>; +}; + +&dp2lvds_backlight0 { + pwms = <&pwm2_8ch_7 0 25000 0>; +}; + +&edp2lvds_backlight0 { + pwms = <&pwm0_2ch_0 0 25000 0>; +}; + +/* edp->serdes->lvds_panel */ +&pwm0_2ch_0 { + pinctrl-0 = <&pwm0m3_ch0>; +}; + +/* dp->serdes->lvds_panel */ +&pwm2_8ch_7 { + pinctrl-0 = <&pwm2m3_ch7>; +}; + +&pinctrl { + touch { + //dsi-i2c8 + touch_gpio_dsi: touch-gpio-dsi { + rockchip,pins = + <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + //dp-i2c5 + touch_gpio_dp: touch-gpio-dp { + rockchip,pins = <4 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + //edp0-i2c3 + touch_gpio_edp: touch-gpio-edp { + rockchip,pins = + <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + vcc5v0-buck { + vcc5v0_buck_en: vcc5v0-buck-en { + rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + cluster-buck { + cluster_buck_en: cluster-buck-en { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; +}; + +&rkvpss { + status = "okay"; +}; + +&rkvpss_mmu { + status = "okay"; +}; + +&rkvpss_vir0 { + status = "okay"; +}; + +&route_dsi { + status = "okay"; +}; + +&sdmmc { + status = "disabled"; +}; + +&ufs { + status = "disabled"; +}; + +&u2phy0 { + status = "okay"; +}; + +&u2phy0_otg { + //vbus-supply = <&usb_otg_vcc5v_buck>; + status = "okay"; +}; + +&u2phy1_otg { + phy-supply = <&usb_host_vcc5v_buck>; + status = "okay"; +}; + +&usbdp_phy { + maximum-speed = "high-speed"; + status = "okay"; +}; + +&usb_drd1_dwc3 { + snps,dis_u2_susphy_quirk; + snps,usb2-lpm-disable; + status = "okay"; +}; From 0439f30ead1393813d69a01587530652908c9171 Mon Sep 17 00:00:00 2001 From: Joseph Chen Date: Fri, 13 Jun 2025 17:19:25 +0800 Subject: [PATCH 26/49] mfd: rk808: Print full chip id of rk801 Signed-off-by: Joseph Chen Change-Id: I195858d23c862d0f961acb4c04bb1102ab9954c9 --- drivers/mfd/rk808.c | 8 ++++++-- include/linux/mfd/rk808.h | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c index 3e9094c8cc0e..d89f979133ad 100644 --- a/drivers/mfd/rk808.c +++ b/drivers/mfd/rk808.c @@ -1400,8 +1400,10 @@ static int rk808_probe(struct i2c_client *client, const struct mfd_cell *cells; u8 on_source = 0, off_source = 0; unsigned int on, off; + u32 pmic_id_mask = RK8XX_ID_MSK; int nr_pre_init_regs; int nr_cells; + int pmic_id; int msb, lsb; unsigned char pmic_id_msb, pmic_id_lsb; int ret; @@ -1423,6 +1425,7 @@ static int rk808_probe(struct i2c_client *client, } else if (of_device_is_compatible(np, "rockchip,rk801")) { pmic_id_msb = RK801_ID_MSB; pmic_id_lsb = RK801_ID_LSB; + pmic_id_mask = RK801_ID_MSK; } else { pmic_id_msb = RK808_ID_MSB; pmic_id_lsb = RK808_ID_LSB; @@ -1443,8 +1446,9 @@ static int rk808_probe(struct i2c_client *client, return lsb; } - rk808->variant = ((msb << 8) | lsb) & RK8XX_ID_MSK; - dev_info(&client->dev, "chip id: 0x%x\n", (unsigned int)rk808->variant); + pmic_id = (msb << 8) | lsb; + rk808->variant = pmic_id & RK8XX_ID_MSK; + dev_info(&client->dev, "chip id: 0x%x\n", pmic_id & pmic_id_mask); switch (rk808->variant) { case RK801_ID: diff --git a/include/linux/mfd/rk808.h b/include/linux/mfd/rk808.h index c374be50e1f2..62e7e33c23e0 100644 --- a/include/linux/mfd/rk808.h +++ b/include/linux/mfd/rk808.h @@ -939,6 +939,7 @@ enum rk805_reg { #define SHUTDOWN_FUN (0x2 << 2) #define SLEEP_FUN (0x1 << 2) #define RK8XX_ID_MSK 0xfff0 +#define RK801_ID_MSK 0xffff #define PWM_MODE_MSK BIT(7) #define FPWM_MODE BIT(7) #define AUTO_PWM_MODE 0 From 502c6719f5bdd1613dfbea205e4da267a89d4ff4 Mon Sep 17 00:00:00 2001 From: Cai Wenzhong Date: Mon, 16 Jun 2025 10:35:41 +0800 Subject: [PATCH 27/49] arm64: dts: rockchip: rk3576-vehicle-evb-v21: add maxim-max96712-dcphy0-ox03j10-dtsi Signed-off-by: Cai Wenzhong Change-Id: I16e902c765d539a365360defd00d3173bc4a3976 --- ...evb-v21-maxim-max96712-dcphy0-ox03j10.dtsi | 729 ++++++++++++++++++ 1 file changed, 729 insertions(+) create mode 100644 arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21-maxim-max96712-dcphy0-ox03j10.dtsi diff --git a/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21-maxim-max96712-dcphy0-ox03j10.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21-maxim-max96712-dcphy0-ox03j10.dtsi new file mode 100644 index 000000000000..cffb04ea4958 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3576-vehicle-evb-v21-maxim-max96712-dcphy0-ox03j10.dtsi @@ -0,0 +1,729 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + * + */ +#include + +/ { + max96712_dcphy0_osc: max96712-dcphy0-oscillator { + compatible = "fixed-clock"; + #clock-cells = <1>; + clock-frequency = <25000000>; + clock-output-names = "max96712-dcphy0-osc"; + }; + + max96712_dcphy0_vcc1v2: max96712-dcphy0-vcc1v2 { + compatible = "regulator-fixed"; + regulator-name = "max96712_dcphy0_vcc1v2"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + startup-delay-us = <850>; + vin-supply = <&vcc_2v0_pldo_s3>; + }; + + max96712_dcphy0_vcc1v8: max96712-dcphy0-vcc1v8 { + compatible = "regulator-fixed"; + regulator-name = "max96712_dcphy0_vcc1v8"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + startup-delay-us = <200>; + vin-supply = <&vcc_2v0_pldo_s3>; + }; + + max96712_dcphy0_pwdn_regulator: max96712-dcphy0-pwdn-regulator { + compatible = "regulator-fixed"; + regulator-name = "max96712_dcphy0_pwdn"; + gpio = <&gpio2 RK_PA0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&max96712_dcphy0_pwdn>; + enable-active-high; + startup-delay-us = <10000>; + off-on-delay-us = <5000>; + }; + + max96712_dcphy0_poc_regulator: max96712-dcphy0-poc-regulator { + compatible = "regulator-fixed"; + regulator-name = "max96712_dcphy0_poc"; + gpio = <&i2c7_nca9539_gpio 2 GPIO_ACTIVE_HIGH>; + enable-active-high; + startup-delay-us = <10000>; + off-on-delay-us = <5000>; + vin-supply = <&dcphy0_vcc12v_buck1>; + }; +}; + +&csi2_dcphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_dcphy0_in_max96712: endpoint@1 { + reg = <1>; + remote-endpoint = <&max96712_dcphy0_out>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidcphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi0_csi2_input>; + }; + }; + }; +}; + +&i2c2 { + max96712_dcphy0: max96712@29 { + compatible = "maxim4c,max96712"; + status = "okay"; + reg = <0x29>; + clock-names = "xvclk"; + clocks = <&max96712_dcphy0_osc 0>; + pinctrl-names = "default"; + pinctrl-0 = <&max96712_dcphy0_errb>, <&max96712_dcphy0_lock>; + power-domains = <&power RK3576_PD_VI>; + rockchip,grf = <&sys_grf>; + vcc1v2-supply = <&max96712_dcphy0_vcc1v2>; + vcc1v8-supply = <&max96712_dcphy0_vcc1v8>; + pwdn-supply = <&max96712_dcphy0_pwdn_regulator>; + lock-gpios = <&gpio2 RK_PA2 GPIO_ACTIVE_HIGH>; + + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + + port { + max96712_dcphy0_out: endpoint { + remote-endpoint = <&mipi_dcphy0_in_max96712>; + data-lanes = <1 2 3 4>; + }; + }; + + /* support mode config start */ + support-mode-config { + status = "okay"; + + bus-format = ; + sensor-width = <1920>; + sensor-height = <1280>; + max-fps-numerator = <10000>; + max-fps-denominator = <300000>; + bpp = <16>; + link-freq-idx = <20>; + }; + /* support mode config end */ + + /* serdes local device start */ + serdes-local-device { + status = "okay"; + + /* GMSL LINK config start */ + gmsl-links { + status = "okay"; + + link-vdd-ldo1-en = <1>; + link-vdd-ldo2-en = <1>; + + // Link A: link-id = 0 + gmsl-link-config-0 { + status = "okay"; + link-id = <0>; // Link ID: 0/1/2/3 + + link-type = <1>; // 0: GMSL1, 1: GMSL2 + link-rx-rate = <1>; // 0: 3GBPS, 1: 6GBPS + link-tx-rate = <0>; // 0: default for 187.5MBPS + + link-remote-cam = <&max96712_dcphy0_cam0>; // remote camera + + link-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 14 D1 03 00 00 // VGAHiGain + 14 45 00 00 00 // Disable SSC + ]; + }; + }; + + // Link B: link-id = 1 + gmsl-link-config-1 { + status = "okay"; + link-id = <1>; // Link ID: 0/1/2/3 + + link-type = <1>; // 0: GMSL1, 1: GMSL2 + link-rx-rate = <1>; // 0: 3GBPS, 1: 6GBPS + link-tx-rate = <0>; // 0: default for 187.5MBPS + + link-remote-cam = <&max96712_dcphy0_cam1>; // remote camera + + link-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 15 D1 03 00 00 // VGAHiGain + 15 45 00 00 00 // Disable SSC + ]; + }; + }; + + // Link C: link-id = 2 + gmsl-link-config-2 { + status = "okay"; + link-id = <2>; // Link ID: 0/1/2/3 + + link-type = <1>; // 0: GMSL1, 1: GMSL2 + link-rx-rate = <1>; // 0: 3GBPS, 1: 6GBPS + link-tx-rate = <0>; // 0: default for 187.5MBPS + + link-remote-cam = <&max96712_dcphy0_cam2>; // remote camera + + link-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 16 D1 03 00 00 // VGAHiGain + 16 45 00 00 00 // Disable SSC + ]; + }; + }; + + // Link D: link-id = 3 + gmsl-link-config-3 { + status = "okay"; + link-id = <3>; // Link ID: 0/1/2/3 + + link-type = <1>; // 0: GMSL1, 1: GMSL2 + link-rx-rate = <1>; // 0: 3GBPS, 1: 6GBPS + link-tx-rate = <0>; // 0: default for 187.5MBPS + + link-remote-cam = <&max96712_dcphy0_cam3>; // remote camera + + link-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 17 D1 03 00 00 // VGAHiGain + 17 45 00 00 00 // Disable SSC + ]; + }; + }; + }; + /* GMSL LINK config end */ + + /* VIDEO PIPE config start */ + video-pipes { + status = "okay"; + + // Video Pipe 0 + video-pipe-config-0 { + status = "okay"; + pipe-id = <0>; // Video Pipe ID: 0/1/2/3/4/5/6/7 + + pipe-idx = <2>; // Video Pipe X/Y/Z/U: 0/1/2/3 + link-idx = <0>; // Link A/B/C/D: 0/1/2/3 + + pipe-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + // Send YUV422, FS, and FE from Video Pipe 0 to Controller 1 + 09 0B 07 00 00 // Enable 0/1/2 SRC/DST Mappings + 09 2D 15 00 00 // SRC/DST 0/1/2 -> CSI2 Controller 1; + // For the following MSB 2 bits = VC, LSB 6 bits = DT + 09 0D 1e 00 00 // SRC0 VC = 0, DT = YUV422 8bit + 09 0E 1e 00 00 // DST0 VC = 0, DT = YUV422 8bit + 09 0F 00 00 00 // SRC1 VC = 0, DT = Frame Start + 09 10 00 00 00 // DST1 VC = 0, DT = Frame Start + 09 11 01 00 00 // SRC2 VC = 0, DT = Frame End + 09 12 01 00 00 // DST2 VC = 0, DT = Frame End + ]; + }; + }; + + // Video Pipe 1 + video-pipe-config-1 { + status = "okay"; + pipe-id = <1>; // Video Pipe 1: pipe-id = 1 + + pipe-idx = <2>; // Video Pipe X/Y/Z/U: 0/1/2/3 + link-idx = <1>; // Link A/B/C/D: 0/1/2/3 + + pipe-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + // Send YUV422, FS, and FE from Video Pipe 1 to Controller 1 + 09 4B 07 00 00 // Enable 0/1/2 SRC/DST Mappings + 09 6D 15 00 00 // SRC/DST 0/1/2 -> CSI2 Controller 1; + // For the following MSB 2 bits = VC, LSB 6 bits = DT + 09 4D 1e 00 00 // SRC0 VC = 0, DT = YUV422 8bit + 09 4E 5e 00 00 // DST0 VC = 1, DT = YUV422 8bit + 09 4F 00 00 00 // SRC1 VC = 0, DT = Frame Start + 09 50 40 00 00 // DST1 VC = 1, DT = Frame Start + 09 51 01 00 00 // SRC2 VC = 0, DT = Frame End + 09 52 41 00 00 // DST2 VC = 1, DT = Frame End + ]; + }; + }; + + // Video Pipe 2 + video-pipe-config-2 { + status = "okay"; + pipe-id = <2>; // Video Pipe ID: 0/1/2/3/4/5/6/7 + + pipe-idx = <2>; // Video Pipe X/Y/Z/U: 0/1/2/3 + link-idx = <2>; // Link A/B/C/D: 0/1/2/3 + + pipe-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + // Send YUV422, FS, and FE from Video Pipe 2 to Controller 1 + 09 8B 07 00 00 // Enable 0/1/2 SRC/DST Mappings + 09 AD 15 00 00 // SRC/DST 0/1/2 -> CSI2 Controller 1; + // For the following MSB 2 bits = VC, LSB 6 bits = DT + 09 8D 1e 00 00 // SRC0 VC = 0, DT = YUV422 8bit + 09 8E 9e 00 00 // DST0 VC = 2, DT = YUV422 8bit + 09 8F 00 00 00 // SRC1 VC = 0, DT = Frame Start + 09 90 80 00 00 // DST1 VC = 2, DT = Frame Start + 09 91 01 00 00 // SRC2 VC = 0, DT = Frame End + 09 92 81 00 00 // DST2 VC = 2, DT = Frame End + ]; + }; + }; + + // Video Pipe 3 + video-pipe-config-3 { + status = "okay"; + pipe-id = <3>; // Video Pipe ID: 0/1/2/3/4/5/6/7 + + pipe-idx = <2>; // Video Pipe X/Y/Z/U: 0/1/2/3 + link-idx = <3>; // Link A/B/C/D: 0/1/2/3 + + pipe-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + // Send YUV422, FS, and FE from Video Pipe 3 to Controller 1 + 09 CB 07 00 00 // Enable 0/1/2 SRC/DST Mappings + 09 ED 15 00 00 // SRC/DST 0/1/2 -> CSI2 Controller 1; + // For the following MSB 2 bits = VC, LSB 6 bits = DT + 09 CD 1e 00 00 // SRC0 VC = 0, DT = YUV422 8bit + 09 CE de 00 00 // DST0 VC = 3, DT = YUV422 8bit + 09 CF 00 00 00 // SRC1 VC = 0, DT = Frame Start + 09 D0 c0 00 00 // DST1 VC = 3, DT = Frame Start + 09 D1 01 00 00 // SRC2 VC = 0, DT = Frame End + 09 D2 c1 00 00 // DST2 VC = 3, DT = Frame End + ]; + }; + }; + }; + /* VIDEO PIPE config end */ + + /* MIPI TXPHY config start */ + mipi-txphys { + status = "okay"; + + phy-mode = <0>; // 0: 4Lanes, 1: 2Lanes + phy-force-clock-out = <1>; // 1: default for force clock out + phy-force-clk0-en = <1>; // provide MIPI clock: 0 = PHY1, 1 = PHY0 + phy-force-clk3-en = <0>; // provide MIPI clock: 0 = PHY2, 1 = PHY3 + + // MIPI TXPHY A: phy-id = 0 + mipi-txphy-config-0 { + status = "okay"; + phy-id = <0>; // MIPI TXPHY ID: 0/1/2/3 + + phy-type = <0>; // 0: DPHY, 1: CPHY + auto-deskew = <0x80>; + data-lane-num = <4>; + data-lane-map = <0x4>; + vc-ext-en = <0>; + }; + + // MIPI TXPHY B: phy-id = 1 + mipi-txphy-config-1 { + status = "okay"; + phy-id = <1>; // MIPI TXPHY ID: 0/1/2/3 + + phy-type = <0>; // 0: DPHY, 1: CPHY + auto-deskew = <0x80>; + data-lane-num = <4>; + data-lane-map = <0xe>; + vc-ext-en = <0>; + }; + }; + /* MIPI TXPHY config end */ + + /* local device extra init sequence */ + extra-init-sequence { + status = "disabled"; + + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + // common init sequence such as fsync / gpio and so on + ]; + }; + }; + /* serdes local device end */ + + /* i2c-mux start */ + i2c-mux { + #address-cells = <1>; + #size-cells = <0>; + + i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + // Note: Serializer node defined before camera node + max96712_dcphy0_ser0: max96717@51 { + compatible = "maxim,ser,max96717"; + reg = <0x51>; + + ser-i2c-addr-def = <0x40>; + + ser-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 02 BE 00 00 01 // MFP0 GPIO_OUT = 0: MFP0 output is driven to 0 + 03 02 10 00 00 // Improve CMU voltage performance to improve link robustness + 14 17 00 00 00 // RLMS17 = 0x00: disable AGC/DFE adaptation + 14 32 7f 00 00 // RLMS32 = 0x7F: change OSN loop mode + 03 F0 59 00 00 // REFGEN_PREDEF_FREQ_ALT = 1: Alternative table, REFGEN_PREDEF_FREQ = 0x1: 24MHz + 00 03 03 00 00 // RCLKSEL = 0x3: Reference PLL output + 00 06 B1 00 01 // RCLKEN = 1: RCLK output is enabled + 02 BF 40 00 00 // MFP0 PULL_UPDN_SEL = 1: Pullup + 02 BE 10 00 01 // MFP0 GPIO_OUT = 1: MFP0 output is driven to 1 + ]; + }; + }; + + max96712_dcphy0_cam0: ox03j10@31 { + compatible = "maxim,ovti,ox03j10"; + reg = <0x31>; + + cam-i2c-addr-def = <0x36>; + + cam-remote-ser = <&max96712_dcphy0_ser0>; // remote serializer + + poc-supply = <&max96712_dcphy0_poc_regulator>; + + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + + /* port config start */ + port { + max96712_dcphy0_cam0_out: endpoint { + data-lanes = <1 2 3 4>; + }; + }; + /* port config end */ + }; + }; + + i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + + // Note: Serializer node defined before camera node + max96712_dcphy0_ser1: max96717@52 { + compatible = "maxim,ser,max96717"; + reg = <0x52>; + + ser-i2c-addr-def = <0x40>; + + ser-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 02 BE 00 00 01 // MFP0 GPIO_OUT = 0: MFP0 output is driven to 0 + 03 02 10 00 00 // Improve CMU voltage performance to improve link robustness + 14 17 00 00 00 // RLMS17 = 0x00: disable AGC/DFE adaptation + 14 32 7f 00 00 // RLMS32 = 0x7F: change OSN loop mode + 03 F0 59 00 00 // REFGEN_PREDEF_FREQ_ALT = 1: Alternative table, REFGEN_PREDEF_FREQ = 0x1: 24MHz + 00 03 03 00 00 // RCLKSEL = 0x3: Reference PLL output + 00 06 B1 00 01 // RCLKEN = 1: RCLK output is enabled + 02 BF 40 00 00 // MFP0 PULL_UPDN_SEL = 1: Pullup + 02 BE 10 00 01 // MFP0 GPIO_OUT = 1: MFP0 output is driven to 1 + ]; + }; + }; + + max96712_dcphy0_cam1: ox03j10@32 { + compatible = "maxim,ovti,ox03j10"; + reg = <0x32>; + + cam-i2c-addr-def = <0x36>; + + cam-remote-ser = <&max96712_dcphy0_ser1>; // remote serializer + + poc-supply = <&max96712_dcphy0_poc_regulator>; + + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + + /* port config start */ + port { + max96712_dcphy0_cam1_out: endpoint { + data-lanes = <1 2 3 4>; + }; + }; + /* port config end */ + }; + }; + + i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <2>; + + // Note: Serializer node defined before camera node + max96712_dcphy0_ser2: max96717@53 { + compatible = "maxim,ser,max96717"; + reg = <0x53>; + + ser-i2c-addr-def = <0x40>; + + ser-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 02 BE 00 00 01 // MFP0 GPIO_OUT = 0: MFP0 output is driven to 0 + 03 02 10 00 00 // Improve CMU voltage performance to improve link robustness + 14 17 00 00 00 // RLMS17 = 0x00: disable AGC/DFE adaptation + 14 32 7f 00 00 // RLMS32 = 0x7F: change OSN loop mode + 03 F0 59 00 00 // REFGEN_PREDEF_FREQ_ALT = 1: Alternative table, REFGEN_PREDEF_FREQ = 0x1: 24MHz + 00 03 03 00 00 // RCLKSEL = 0x3: Reference PLL output + 00 06 B1 00 01 // RCLKEN = 1: RCLK output is enabled + 02 BF 40 00 00 // MFP0 PULL_UPDN_SEL = 1: Pullup + 02 BE 10 00 01 // MFP0 GPIO_OUT = 1: MFP0 output is driven to 1 + ]; + }; + }; + + max96712_dcphy0_cam2: ox03j10@33 { + compatible = "maxim,ovti,ox03j10"; + reg = <0x33>; + + cam-i2c-addr-def = <0x36>; + + cam-remote-ser = <&max96712_dcphy0_ser2>; // remote serializer + + poc-supply = <&max96712_dcphy0_poc_regulator>; + + rockchip,camera-module-index = <2>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + + /* port config start */ + port { + max96712_dcphy0_cam2_out: endpoint { + data-lanes = <1 2 3 4>; + }; + }; + /* port config end */ + }; + }; + + i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + reg = <3>; + + // Note: Serializer node defined before camera node + max96712_dcphy0_ser3: max96717@54 { + compatible = "maxim,ser,max96717"; + reg = <0x54>; + + ser-i2c-addr-def = <0x40>; + + ser-init-sequence { + seq-item-size = <5>; // reg-addr-len + reg-val-len * 2 + 1 + reg-addr-len = <2>; // 1: 8bits, 2: 16bits + reg-val-len = <1>; // 1: 8bits, 2: 16bits, 3: 24bits + + // reg_addr reg_val val_mask delay + init-sequence = [ + 02 BE 00 00 01 // MFP0 GPIO_OUT = 0: MFP0 output is driven to 0 + 03 02 10 00 00 // Improve CMU voltage performance to improve link robustness + 14 17 00 00 00 // RLMS17 = 0x00: disable AGC/DFE adaptation + 14 32 7f 00 00 // RLMS32 = 0x7F: change OSN loop mode + 03 F0 59 00 00 // REFGEN_PREDEF_FREQ_ALT = 1: Alternative table, REFGEN_PREDEF_FREQ = 0x1: 24MHz + 00 03 03 00 00 // RCLKSEL = 0x3: Reference PLL output + 00 06 B1 00 01 // RCLKEN = 1: RCLK output is enabled + 02 BF 40 00 00 // MFP0 PULL_UPDN_SEL = 1: Pullup + 02 BE 10 00 01 // MFP0 GPIO_OUT = 1: MFP0 output is driven to 1 + ]; + }; + }; + + max96712_dcphy0_cam3: ox03j10@34 { + compatible = "maxim,ovti,ox03j10"; + reg = <0x34>; + + cam-i2c-addr-def = <0x36>; + + cam-remote-ser = <&max96712_dcphy0_ser3>; // remote serializer + + poc-supply = <&max96712_dcphy0_poc_regulator>; + + rockchip,camera-module-index = <3>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + + /* port config start */ + port { + max96712_dcphy0_cam3_out: endpoint { + data-lanes = <1 2 3 4>; + }; + }; + /* port config end */ + }; + }; + }; + /* i2c-mux end */ + }; +}; + +&mipi0_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&csidcphy0_out>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi0_in>; + }; + }; + }; +}; + +&rkcif_mipi_lvds { + status = "okay"; + /* parameters for do cif reset detecting: + * index0: monitor mode, + 0 for idle, + 1 for continue, + 2 for trigger, + 3 for hotplug (for nextchip) + * index1: the frame id to start timer, + min is 2 + * index2: frame num of monitoring cycle + * index3: err time for keep monitoring + after finding out err (ms) + * index4: csi2 err reference val for resetting + */ + rockchip,cif-monitor = <3 2 1 1000 5>; + + port { + cif_mipi0_in: endpoint { + remote-endpoint = <&mipi0_csi2_output>; + }; + }; +}; + +&rkcif { + status = "okay"; + rockchip,android-usb-camerahal-enable; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&pinctrl { + max96712-dcphy0 { + max96712_dcphy0_pwdn: max96712-dcphy0-pwdn { + rockchip,pins = <2 RK_PA0 RK_FUNC_GPIO &pcfg_output_low>; + }; + + max96712_dcphy0_errb: max96712-dcphy0-errb { + rockchip,pins = <2 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none_smt>; + }; + + max96712_dcphy0_lock: max96712-dcphy0-lock { + rockchip,pins = <2 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none_smt>; + }; + }; +}; From 9d76e5f54d8624d5dc489131edb6c0d09fd04c94 Mon Sep 17 00:00:00 2001 From: Yuefu Su Date: Fri, 13 Jun 2025 18:48:30 +0800 Subject: [PATCH 28/49] arm64: dts: rockchip: Add v1126b-thunder-boot-cam.dtsi Signed-off-by: Yuefu Su Change-Id: Ia5405cfe7ffe549bb59df495bcc5dbebe64ac826 --- .../rockchip/rv1126b-thunder-boot-cam.dtsi | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot-cam.dtsi diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot-cam.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot-cam.dtsi new file mode 100644 index 000000000000..1a4eb65902e2 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126b-thunder-boot-cam.dtsi @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + * + */ + +&csi2_dphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + csi_dphy_input0: endpoint@1 { + reg = <1>; + remote-endpoint = <&cam0_out>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi0_csi2_input>; + }; + }; + }; +}; + +&mipi0_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&csidphy0_out>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi_in0>; + }; + }; + }; +}; + +&rkcif { + status = "okay"; +}; + +&rkcif_mipi_lvds { + status = "okay"; + memory-region-thunderboot = <&rkisp_thunderboot>; + + port { + cif_mipi_in0: endpoint { + remote-endpoint = <&mipi0_csi2_output>; + }; + }; +}; + +&rkcif_mipi_lvds_sditf { + status = "okay"; + + port { + mipi_lvds_sditf: endpoint { + remote-endpoint = <&isp_vir0>; + }; + }; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + memory-region-thunderboot = <&rkisp_thunderboot>; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_vir0: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi_lvds_sditf>; + }; + }; +}; + +&rkisp_vir0_sditf { + status = "okay"; +}; + +&rkvpss { + status = "okay"; + dvbm = <&rkdvbm>; +}; + +&rkvpss_mmu { + status = "okay"; +}; + +&rkvpss_vir0 { + status = "okay"; +}; From 4cb57aab9fc9fdcef3e3dcf9fd5ff1810bd1ae65 Mon Sep 17 00:00:00 2001 From: Yuefu Su Date: Fri, 13 Jun 2025 18:49:55 +0800 Subject: [PATCH 29/49] arm64: dts: rockchip: rv1126b-evb2-v10-tb-400w: Add camera support Signed-off-by: Yuefu Su Change-Id: Ibe6bec939c40cfd9b7ef4ca5b115d2ed0fa9c81f --- .../dts/rockchip/rv1126b-evb2-v10-tb-400w.dts | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-tb-400w.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-tb-400w.dts index de9f0fecf79f..08b6a95d551d 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-tb-400w.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-tb-400w.dts @@ -3,7 +3,11 @@ * Copyright (c) 2025 Rockchip Electronics Co., Ltd. */ -#include "rv1126b-evb2-v10.dts" +/dts-v1/; +#include "rv1126b.dtsi" +#include "rv1126b-evb.dtsi" +#include "rv1126b-evb2-v10.dtsi" +#include "rv1126b-thunder-boot-cam.dtsi" #include "rv1126b-thunder-boot-spi-nor.dtsi" / { @@ -15,6 +19,36 @@ }; }; +&i2c3 { + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m1_pins>; + rockchip,amp-shared; + status = "okay"; + + sc450ai: sc450ai@30 { + compatible = "smartsens,sc450ai"; + status = "okay"; + reg = <0x30>; + clocks = <&cru CLK_MIPI0_OUT2IO>; + clock-names = "xvclk"; + reset-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_LOW>; + pwdn-gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clk0_pins>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "default"; + rockchip,camera-module-lens-name = "default"; + port { + cam0_out: endpoint { + remote-endpoint = <&csi_dphy_input0>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + &ramdisk_r { reg = <0x48c40000 (40 * 0x00100000)>; }; @@ -22,3 +56,12 @@ &ramdisk_c { reg = <0x4b440000 (20 * 0x00100000)>; }; + +&rkisp_thunderboot { + /* reg's offset MUST match with RTOS */ + /* + * vicap, capture raw10, ceil(w*10/8/256)*256*h *4(buf num) + * e.g. 2688x1520: 0x14c8000 + */ + reg = <0x41300000 0x14c8000>; +}; From c0048603a00ce0d6ffc8ebea153078e609a38236 Mon Sep 17 00:00:00 2001 From: Weiwen Chen Date: Mon, 16 Jun 2025 11:58:54 +0800 Subject: [PATCH 30/49] ARM: configs: rv1126b-rndis support rndis config Signed-off-by: Weiwen Chen Change-Id: Ic03129a42e63385d404f46c523b7d0c6c6a33929 --- arch/arm/configs/rv1126b-rndis.config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm/configs/rv1126b-rndis.config b/arch/arm/configs/rv1126b-rndis.config index 54cdea391359..9c0c5e431e4b 100644 --- a/arch/arm/configs/rv1126b-rndis.config +++ b/arch/arm/configs/rv1126b-rndis.config @@ -68,7 +68,7 @@ CONFIG_USB_CONFIGFS_F_UVC=y # CONFIG_USB_CONFIGFS_MASS_STORAGE is not set # CONFIG_USB_CONFIGFS_NCM is not set # CONFIG_USB_CONFIGFS_OBEX is not set -# CONFIG_USB_CONFIGFS_RNDIS is not set +CONFIG_USB_CONFIGFS_RNDIS=y # CONFIG_USB_CONFIGFS_SERIAL is not set CONFIG_USB_CONFIGFS_UEVENT=y # CONFIG_USB_CONN_GPIO is not set @@ -101,6 +101,7 @@ CONFIG_USB_EHCI_TT_NEWSCHED=y # CONFIG_USB_FUSB300 is not set CONFIG_USB_F_FS=m CONFIG_USB_F_HID=m +CONFIG_USB_F_RNDIS=m CONFIG_USB_F_UAC1=m CONFIG_USB_F_UAC2=m CONFIG_USB_F_UVC=m @@ -170,6 +171,7 @@ CONFIG_USB_ROLE_SWITCH=m # CONFIG_USB_ULPI is not set # CONFIG_USB_ULPI_BUS is not set CONFIG_USB_U_AUDIO=m +CONFIG_USB_U_ETHER=m # CONFIG_USB_WDM is not set # CONFIG_USB_XHCI_DBGCAP is not set CONFIG_USB_XHCI_HCD=m From bb36a2c6523c09aa8d5502be380bfa6a86651e9c Mon Sep 17 00:00:00 2001 From: Oliver Peng Date: Thu, 12 Jun 2025 07:00:39 +0000 Subject: [PATCH 31/49] arm64: dts: rockchip: Add rk3576 toybrick SD0 board Signed-off-by: Oliver Peng Change-Id: I85ee487c1b960fc05e84e40a5973eef0a2615f78 --- arch/arm64/boot/dts/rockchip/Makefile | 1 + .../rockchip/rk3576-toybrick-cam-dcphy0.dtsi | 195 +++ .../rockchip/rk3576-toybrick-cam-dphy3.dtsi | 199 +++ .../dts/rockchip/rk3576-toybrick-d0-linux.dts | 1444 +++++++++++++++++ 4 files changed, 1839 insertions(+) create mode 100644 arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dcphy0.dtsi create mode 100644 arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dphy3.dtsi create mode 100644 arch/arm64/boot/dts/rockchip/rk3576-toybrick-d0-linux.dts diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile index 72a54df04cfe..ecaecd996374 100644 --- a/arch/arm64/boot/dts/rockchip/Makefile +++ b/arch/arm64/boot/dts/rockchip/Makefile @@ -293,6 +293,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-test2-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-test2-v10-linux.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-test3-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-test5-v10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-toybrick-d0-linux.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v10.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v10-linux.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-vehicle-evb-v20.dtb diff --git a/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dcphy0.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dcphy0.dtsi new file mode 100644 index 000000000000..f1e4b25a20c9 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dcphy0.dtsi @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + */ + +&csi2_dcphy0 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam0: endpoint@1 { + reg = <1>; + remote-endpoint = <&imx415_out0>; + data-lanes = <1 2 3 4>; + }; + + mipi_in_ucam1: endpoint@2 { + reg = <2>; + remote-endpoint = <&ov50c40_out0>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidcphy0_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi0_csi2_input>; + }; + }; + }; +}; + +&i2c9 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c9m2_xfer>; + + imx415_dcphy0: imx415@1a { + compatible = "sony,imx415"; + reg = <0x1a>; + clocks = <&cru CLK_MIPI_CAMERAOUT_M0>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clk0m1_clk0>; + power-domains = <&power RK3576_PD_VI>; + power-gpios = <&gpio2 RK_PD0 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT2022-PX1"; + rockchip,camera-module-lens-name = "IR0147-50IRC-8M-F20"; + port { + imx415_out0: endpoint { + remote-endpoint = <&mipi_in_ucam0>; + data-lanes = <1 2 3 4>; + }; + }; + }; + + aw8601_0: aw8601@c { + compatible = "awinic,aw8601"; + status = "okay"; + reg = <0x0c>; + rockchip,vcm-start-current = <56>; + rockchip,vcm-rated-current = <96>; + rockchip,vcm-step-mode = <4>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + otp_eeprom_0: otp_eeprom@50 { + compatible = "rk,otp_eeprom"; + status = "okay"; + reg = <0x50>; + }; + + ov50c40_dcph0: ov50c40@36 { + compatible = "ovti,ov50c40"; + reg = <0x36>; + clocks = <&cru CLK_MIPI_CAMERAOUT_M0>; + clock-names = "xvclk"; + power-domains = <&power RK3576_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clk0m1_clk0>; + pwdn-gpios = <&gpio2 RK_PD0 GPIO_ACTIVE_LOW>;// must be high at last + reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>;// must be high at last + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "HZGA06"; + rockchip,camera-module-lens-name = "ZE0082C1-RK3568"; + eeprom-ctrl = <&otp_eeprom_0>; + lens-focus = <&aw8601_0>; + port { + ov50c40_out0: endpoint { + remote-endpoint = <&mipi_in_ucam1>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&mipi0_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&csidcphy0_out>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi0_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi_in0>; + }; + }; + }; +}; + +&rkcif { + status = "okay"; +}; + +&rkcif_mipi_lvds { + status = "okay"; + + port { + cif_mipi_in0: endpoint { + remote-endpoint = <&mipi0_csi2_output>; + }; + }; +}; + +&rkcif_mipi_lvds_sditf { + status = "okay"; + + port { + mipi_lvds_sditf: endpoint { + remote-endpoint = <&isp_vir0>; + }; + }; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir0 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_vir0: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi_lvds_sditf>; + }; + }; +}; + +&rkisp_vir0_sditf { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dphy3.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dphy3.dtsi new file mode 100644 index 000000000000..031c6e160f44 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-cam-dphy3.dtsi @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + */ + +&csi2_dphy0_hw { + status = "okay"; +}; + +&csi2_dphy1_hw { + status = "okay"; +}; + +&csi2_dphy3 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi_in_ucam30: endpoint@1 { + reg = <1>; + remote-endpoint = <&imx415_out1>; + data-lanes = <1 2 3 4>; + }; + + mipi_in_ucam31: endpoint@2 { + reg = <2>; + remote-endpoint = <&ov50c40_out1>; + data-lanes = <1 2 3 4>; + }; + }; + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + csidphy3_out: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi3_csi2_input>; + }; + }; + }; +}; + +&i2c6 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c6m1_xfer>; + + imx415_dphy3: imx415@1a { + compatible = "sony,imx415"; + reg = <0x1a>; + clocks = <&cru CLK_MIPI_CAMERAOUT_M2>; + clock-names = "xvclk"; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clk2m1_clk2>; + power-domains = <&power RK3576_PD_VI>; + power-gpios = <&gpio1 RK_PD4 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_LOW>; + rockchip,camera-module-index = <1>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "CMK-OT2022-PX1"; + rockchip,camera-module-lens-name = "IR0147-50IRC-8M-F20"; + port { + imx415_out1: endpoint { + remote-endpoint = <&mipi_in_ucam30>; + data-lanes = <1 2 3 4>; + }; + }; + }; + + aw8601_1: aw8601@c { + compatible = "awinic,aw8601"; + status = "okay"; + reg = <0x0c>; + rockchip,vcm-start-current = <56>; + rockchip,vcm-rated-current = <96>; + rockchip,vcm-step-mode = <4>; + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + }; + + otp_eeprom_1: otp_eeprom@50 { + compatible = "rk,otp_eeprom"; + status = "okay"; + reg = <0x50>; + }; + + ov50c40_dphy3: ov50c40@36 { + compatible = "ovti,ov50c40"; + reg = <0x36>; + clocks = <&cru CLK_MIPI_CAMERAOUT_M2>; + clock-names = "xvclk"; + power-domains = <&power RK3576_PD_VI>; + pinctrl-names = "default"; + pinctrl-0 = <&cam_clk2m1_clk2>; + pwdn-gpios = <&gpio1 RK_PD4 GPIO_ACTIVE_LOW>;// must be high at last + reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_LOW>;// must be high at last + rockchip,camera-module-index = <0>; + rockchip,camera-module-facing = "back"; + rockchip,camera-module-name = "HZGA06"; + rockchip,camera-module-lens-name = "ZE0082C1-RK3568"; + eeprom-ctrl = <&otp_eeprom_1>; + lens-focus = <&aw8601_1>; + port { + ov50c40_out1: endpoint { + remote-endpoint = <&mipi_in_ucam31>; + data-lanes = <1 2 3 4>; + }; + }; + }; +}; + +&mipi3_csi2 { + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + mipi3_csi2_input: endpoint@1 { + reg = <1>; + remote-endpoint = <&csidphy3_out>; + }; + }; + + port@1 { + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + mipi3_csi2_output: endpoint@0 { + reg = <0>; + remote-endpoint = <&cif_mipi3_in>; + }; + }; + }; +}; + +&rkcif { + status = "okay"; +}; + +&rkcif_mipi_lvds3 { + status = "okay"; + + port { + cif_mipi3_in: endpoint { + remote-endpoint = <&mipi3_csi2_output>; + }; + }; +}; + +&rkcif_mipi_lvds3_sditf { + status = "okay"; + + port { + mipi_lvds3_sditf: endpoint { + remote-endpoint = <&isp_vir1>; + }; + }; +}; + +&rkcif_mmu { + status = "okay"; +}; + +&rkisp { + status = "okay"; +}; + +&rkisp_mmu { + status = "okay"; +}; + +&rkisp_vir1 { + status = "okay"; + + port { + #address-cells = <1>; + #size-cells = <0>; + + isp_vir1: endpoint@0 { + reg = <0>; + remote-endpoint = <&mipi_lvds3_sditf>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-toybrick-d0-linux.dts b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-d0-linux.dts new file mode 100644 index 000000000000..ca214d45edf3 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3576-toybrick-d0-linux.dts @@ -0,0 +1,1444 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + */ + +/dts-v1/; + +#include +#include +#include +#include +#include +#include +#include +#include +#include "rk3576.dtsi" +#include "rk3576-linux.dtsi" +#include "rk3576-pinctrl.dtsi" +#include "rk3576-toybrick-cam-dcphy0.dtsi" +#include "rk3576-toybrick-cam-dphy3.dtsi" + +/ { + model = "Rockchip RK3576 TOYBRICK D0 Board"; + compatible = "rockchip,rk3576-toybrick-d0", "rockchip,rk3576"; + + reserved_memory: reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + vendor_storage_rm: vendor-storage-rm@00000000 { + compatible = "rockchip,vendor-storage-rm"; + reg = <0x0 0x0 0x0 0x0>; + }; + }; + vendor_storage: vendor-storage { + compatible = "rockchip,ram-vendor-storage"; + memory-region = <&vendor_storage_rm>; + status = "okay"; + }; + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm1_6ch_1 0 25000 0>; + brightness-levels = < + 0 20 20 21 21 22 22 23 + 23 24 24 25 25 26 26 27 + 27 28 28 29 29 30 30 31 + 31 32 32 33 33 34 34 35 + 35 36 36 37 37 38 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255 + >; + default-brightness-level = <200>; + }; + + fan1 { + compatible = "pwm-fan"; + pwms = <&pwm1_6ch_4 0 40000 0>; + cooling-levels = <0 80 170 255>; + }; + + // leds: gpio-leds { + // compatible = "gpio-leds"; + // pinctrl-names = "default"; + // pinctrl-0 =<&leds_gpio>; + + // led@1 { + // gpios = <&gpio3 16 GPIO_ACTIVE_HIGH>; + // label = "heartbeat"; + // linux,default-trigger = "heartbeat"; + // }; + // }; + + leds: leds { + compatible = "gpio-leds"; + work_led: work { + gpios = <&gpio3 16 GPIO_ACTIVE_HIGH>; + label = "heartbeat"; + linux,default-trigger = "heartbeat"; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&hym8563>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_poweren_gpio>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio2 RK_PC0 GPIO_ACTIVE_LOW>; + }; + + vbus_typec_power: vbus-typec-power { + compatible = "regulator-fixed"; + regulator-name = "vbus_typec_power"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + }; + + vcc_5v0_sys_s5: vcc-5v0-sys-s5 { + compatible = "regulator-fixed"; + regulator-name = "vcc_5v0_sys_s5"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vbus_typec_power>; + }; + + vcc_2v0_pldo_s3: vcc-2v0-pldo-s3 { + compatible = "regulator-fixed"; + regulator-name = "vcc_2v0_pldo_s3"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <2000000>; + regulator-max-microvolt = <2000000>; + vin-supply = <&vcc_5v0_sys_s5>; + }; + + vcc_1v1_nldo_s3: vcc-1v1-nldo-s3 { + compatible = "regulator-fixed"; + regulator-name = "vcc_1v1_nldo_s3"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + vin-supply = <&vcc_5v0_sys_s5>; + }; + + vcc5v0_usb30_host1: vcc5v0-usb30-host1 { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_usb30_host1"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vcc_5v0_sys_s5>; + enable-active-high; + gpio = <&gpio0 RK_PD3 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&usb_host_pwren>; + }; + + // vcc5v0_usb_otg0: vcc5v0-usb-otg0 { + // compatible = "regulator-fixed"; + // regulator-name = "vcc5v0_usb_otg0"; + // regulator-boot-on; + // regulator-always-on; + // regulator-min-microvolt = <5000000>; + // regulator-max-microvolt = <5000000>; + // vin-supply = <&vcc_5v0_sys_s5>; + // enable-active-high; + // gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; + // pinctrl-names = "default"; + // pinctrl-0 = <&usb_otg_pwren>; + // }; + + vcc_3v3_sd: vcc-3v3-sd { + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3_sd"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3_s3>; + enable-active-high; + gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&sd_pwren>; + }; + + vcc_3v3_pcie: vcc-3v3-pcie { + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3_pcie"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_5v0_sys_s5>; + enable-active-high; + gpio = <&gpio2 RK_PC7 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_pwren>; + }; + + vcc_3v3_s0: vcc-3v3-s0 { + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3_s0"; + regulator-boot-on; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3_s3>; + }; + + vcc3v3_lcd_n: vcc3v3-lcd0-n { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_lcd0_n"; + regulator-boot-on; + enable-active-high; + gpio = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>; + vin-supply = <&vcc_3v3_s0>; + }; + + wireless_bluetooth: wireless-bluetooth { + compatible = "bluetooth-platdata"; + uart_rts_gpios = <&gpio2 RK_PB5 GPIO_ACTIVE_LOW>; + pinctrl-names = "default", "rts_gpio"; + pinctrl-0 = <&uart7m0_rtsn>, <&bt_reset_gpio>, <&bt_host_wake_irq>, <&bt_wake_gpio>; + pinctrl-1 = <&uart7_gpios>; + BT,reset_gpio = <&gpio2 RK_PC1 GPIO_ACTIVE_HIGH>; + BT,wake_gpio = <&gpio2 RK_PC6 GPIO_ACTIVE_HIGH>; + BT,wake_host_irq = <&gpio0 RK_PB1 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; + + wireless_wlan: wireless-wlan { + compatible = "wlan-platdata"; + wifi_chip_type = "ap6275p"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_irq>; + WIFI,host_wake_irq = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; + // WIFI,poweren_gpio = <&gpio2 RK_PC0 GPIO_ACTIVE_HIGH>; + status = "okay"; + }; +}; + +&combphy0_ps { + status = "okay"; +}; + +&combphy1_psu { + status = "okay"; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_lit_s0>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_big_s0>; +}; + +&display_subsystem { + clocks = <&hdptxphy_hdmi>; + clock-names = "hdmi0_phy_pll"; +}; + +&dsi { + status = "disabled"; + //rockchip,lane-rate = <1000>; + dsi_panel: panel@0 { + status = "okay"; + compatible = "simple-panel-dsi"; + reg = <0>; + power-supply = <&vcc3v3_lcd_n>; + backlight = <&backlight>; + reset-delay-ms = <10>; + enable-delay-ms = <10>; + prepare-delay-ms = <10>; + unprepare-delay-ms = <10>; + disable-delay-ms = <60>; + width-mm = <68>; + height-mm = <121>; + dsi,flags = <(MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | + MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_NO_EOT_PACKET)>; + dsi,format = ; + dsi,lanes = <4>; + panel-init-sequence = [ + 23 00 02 FE 21 + 23 00 02 04 00 + 23 00 02 00 64 + 23 00 02 2A 00 + 23 00 02 26 64 + 23 00 02 54 00 + 23 00 02 50 64 + 23 00 02 7B 00 + 23 00 02 77 64 + 23 00 02 A2 00 + 23 00 02 9D 64 + 23 00 02 C9 00 + 23 00 02 C5 64 + 23 00 02 01 71 + 23 00 02 27 71 + 23 00 02 51 71 + 23 00 02 78 71 + 23 00 02 9E 71 + 23 00 02 C6 71 + 23 00 02 02 89 + 23 00 02 28 89 + 23 00 02 52 89 + 23 00 02 79 89 + 23 00 02 9F 89 + 23 00 02 C7 89 + 23 00 02 03 9E + 23 00 02 29 9E + 23 00 02 53 9E + 23 00 02 7A 9E + 23 00 02 A0 9E + 23 00 02 C8 9E + 23 00 02 09 00 + 23 00 02 05 B0 + 23 00 02 31 00 + 23 00 02 2B B0 + 23 00 02 5A 00 + 23 00 02 55 B0 + 23 00 02 80 00 + 23 00 02 7C B0 + 23 00 02 A7 00 + 23 00 02 A3 B0 + 23 00 02 CE 00 + 23 00 02 CA B0 + 23 00 02 06 C0 + 23 00 02 2D C0 + 23 00 02 56 C0 + 23 00 02 7D C0 + 23 00 02 A4 C0 + 23 00 02 CB C0 + 23 00 02 07 CF + 23 00 02 2F CF + 23 00 02 58 CF + 23 00 02 7E CF + 23 00 02 A5 CF + 23 00 02 CC CF + 23 00 02 08 DD + 23 00 02 30 DD + 23 00 02 59 DD + 23 00 02 7F DD + 23 00 02 A6 DD + 23 00 02 CD DD + 23 00 02 0E 15 + 23 00 02 0A E9 + 23 00 02 36 15 + 23 00 02 32 E9 + 23 00 02 5F 15 + 23 00 02 5B E9 + 23 00 02 85 15 + 23 00 02 81 E9 + 23 00 02 AD 15 + 23 00 02 A9 E9 + 23 00 02 D3 15 + 23 00 02 CF E9 + 23 00 02 0B 14 + 23 00 02 33 14 + 23 00 02 5C 14 + 23 00 02 82 14 + 23 00 02 AA 14 + 23 00 02 D0 14 + 23 00 02 0C 36 + 23 00 02 34 36 + 23 00 02 5D 36 + 23 00 02 83 36 + 23 00 02 AB 36 + 23 00 02 D1 36 + 23 00 02 0D 6B + 23 00 02 35 6B + 23 00 02 5E 6B + 23 00 02 84 6B + 23 00 02 AC 6B + 23 00 02 D2 6B + 23 00 02 13 5A + 23 00 02 0F 94 + 23 00 02 3B 5A + 23 00 02 37 94 + 23 00 02 64 5A + 23 00 02 60 94 + 23 00 02 8A 5A + 23 00 02 86 94 + 23 00 02 B2 5A + 23 00 02 AE 94 + 23 00 02 D8 5A + 23 00 02 D4 94 + 23 00 02 10 D1 + 23 00 02 38 D1 + 23 00 02 61 D1 + 23 00 02 87 D1 + 23 00 02 AF D1 + 23 00 02 D5 D1 + 23 00 02 11 04 + 23 00 02 39 04 + 23 00 02 62 04 + 23 00 02 88 04 + 23 00 02 B0 04 + 23 00 02 D6 04 + 23 00 02 12 05 + 23 00 02 3A 05 + 23 00 02 63 05 + 23 00 02 89 05 + 23 00 02 B1 05 + 23 00 02 D7 05 + 23 00 02 18 AA + 23 00 02 14 36 + 23 00 02 42 AA + 23 00 02 3D 36 + 23 00 02 69 AA + 23 00 02 65 36 + 23 00 02 8F AA + 23 00 02 8B 36 + 23 00 02 B7 AA + 23 00 02 B3 36 + 23 00 02 DD AA + 23 00 02 D9 36 + 23 00 02 15 74 + 23 00 02 3F 74 + 23 00 02 66 74 + 23 00 02 8C 74 + 23 00 02 B4 74 + 23 00 02 DA 74 + 23 00 02 16 9F + 23 00 02 40 9F + 23 00 02 67 9F + 23 00 02 8D 9F + 23 00 02 B5 9F + 23 00 02 DB 9F + 23 00 02 17 DC + 23 00 02 41 DC + 23 00 02 68 DC + 23 00 02 8E DC + 23 00 02 B6 DC + 23 00 02 DC DC + 23 00 02 1D FF + 23 00 02 19 03 + 23 00 02 47 FF + 23 00 02 43 03 + 23 00 02 6E FF + 23 00 02 6A 03 + 23 00 02 94 FF + 23 00 02 90 03 + 23 00 02 BC FF + 23 00 02 B8 03 + 23 00 02 E2 FF + 23 00 02 DE 03 + 23 00 02 1A 35 + 23 00 02 44 35 + 23 00 02 6B 35 + 23 00 02 91 35 + 23 00 02 B9 35 + 23 00 02 DF 35 + 23 00 02 1B 45 + 23 00 02 45 45 + 23 00 02 6C 45 + 23 00 02 92 45 + 23 00 02 BA 45 + 23 00 02 E0 45 + 23 00 02 1C 55 + 23 00 02 46 55 + 23 00 02 6D 55 + 23 00 02 93 55 + 23 00 02 BB 55 + 23 00 02 E1 55 + 23 00 02 22 FF + 23 00 02 1E 68 + 23 00 02 4C FF + 23 00 02 48 68 + 23 00 02 73 FF + 23 00 02 6F 68 + 23 00 02 99 FF + 23 00 02 95 68 + 23 00 02 C1 FF + 23 00 02 BD 68 + 23 00 02 E7 FF + 23 00 02 E3 68 + 23 00 02 1F 7E + 23 00 02 49 7E + 23 00 02 70 7E + 23 00 02 96 7E + 23 00 02 BE 7E + 23 00 02 E4 7E + 23 00 02 20 97 + 23 00 02 4A 97 + 23 00 02 71 97 + 23 00 02 97 97 + 23 00 02 BF 97 + 23 00 02 E5 97 + 23 00 02 21 B5 + 23 00 02 4B B5 + 23 00 02 72 B5 + 23 00 02 98 B5 + 23 00 02 C0 B5 + 23 00 02 E6 B5 + 23 00 02 25 F0 + 23 00 02 23 E8 + 23 00 02 4F F0 + 23 00 02 4D E8 + 23 00 02 76 F0 + 23 00 02 74 E8 + 23 00 02 9C F0 + 23 00 02 9A E8 + 23 00 02 C4 F0 + 23 00 02 C2 E8 + 23 00 02 EA F0 + 23 00 02 E8 E8 + 23 00 02 24 FF + 23 00 02 4E FF + 23 00 02 75 FF + 23 00 02 9B FF + 23 00 02 C3 FF + 23 00 02 E9 FF + 23 00 02 FE 3D + 23 00 02 00 04 + 23 00 02 FE 23 + 23 00 02 08 82 + 23 00 02 0A 00 + 23 00 02 0B 00 + 23 00 02 0C 01 + 23 00 02 16 00 + 23 00 02 18 02 + 23 00 02 1B 04 + 23 00 02 19 04 + 23 00 02 1C 81 + 23 00 02 1F 00 + 23 00 02 20 03 + 23 00 02 23 04 + 23 00 02 21 01 + 23 00 02 54 63 + 23 00 02 55 54 + 23 00 02 6E 45 + 23 00 02 6D 36 + 23 00 02 FE 3D + 23 00 02 55 78 + 23 00 02 FE 20 + 23 00 02 26 30 + 23 00 02 FE 3D + 23 00 02 20 71 + 23 00 02 50 8F + 23 00 02 51 8F + 23 00 02 FE 00 + 23 00 02 35 00 + 05 78 01 11 + 05 00 01 29 + ]; + + panel-exit-sequence = [ + 05 00 01 28 + 05 00 01 10 + ]; + + disp_timings0: display-timings { + native-mode = <&dsi_timing0>; + dsi_timing0: timing0 { + clock-frequency = <132000000>; + hactive = <1080>; + vactive = <1920>; + hfront-porch = <15>; + hsync-len = <4>; + hback-porch = <30>; + vfront-porch = <15>; + vsync-len = <2>; + vback-porch = <15>; + hsync-active = <0>; + vsync-active = <0>; + de-active = <0>; + pixelclk-active = <0>; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + panel_in_dsi: endpoint { + remote-endpoint = <&dsi_out_panel>; + }; + }; + }; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + dsi_out_panel: endpoint { + remote-endpoint = <&panel_in_dsi>; + }; + }; + }; + +}; + +&dsi_in_vp0 { + status = "disabled"; +}; + +&dsi_in_vp1 { + status = "disabled"; +}; + +&dsi_in_vp2 { + status = "disabled"; +}; + +&dsi_in_vopl { + status = "disabled"; +}; + +&gmac0 { + /* Use rgmii-rxid mode to disable rx delay inside Soc */ + phy-mode = "rgmii-rxid"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + pinctrl-names = "default"; + pinctrl-0 = <ð0m0_miim + ð0m0_tx_bus2 + ð0m0_rx_bus2 + ð0m0_rgmii_clk + ð0m0_rgmii_bus + ðm0_clk0_25m_out>; + + tx_delay = <0x21>; + /* rx_delay = <0x3f>; */ + + phy-handle = <&rgmii_phy0>; + status = "okay"; +}; + +&gmac1 { + /* Use rgmii-rxid mode to disable rx delay inside Soc */ + phy-mode = "rgmii-rxid"; + clock_in_out = "output"; + + snps,reset-gpio = <&gpio3 RK_PB0 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + /* Reset time is 20ms, 100ms for rtl8211f */ + snps,reset-delays-us = <0 20000 100000>; + + pinctrl-names = "default"; + pinctrl-0 = <ð1m1_miim + ð1m1_tx_bus2 + ð1m1_rx_bus2 + ð1m1_rgmii_clk + ð1m1_rgmii_bus + ðm1_clk1_25m_out>; + + tx_delay = <0x22>; + /* rx_delay = <0x3f>; */ + + phy-handle = <&rgmii_phy1>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_gpu_s0>; + status = "okay"; +}; + +&hdmi { + status = "okay"; + enable-gpios = <&gpio4 RK_PC4 GPIO_ACTIVE_HIGH>; + rockchip,sda-falling-delay-ns = <360>; +}; + +&hdmi_in_vp0 { + status = "okay"; +}; + +&hdptxphy_hdmi { + status = "okay"; +}; + +&i2c0 { + pinctrl-0 = <&i2c0m1_xfer>; + status = "okay"; + + gt1x: gt1x@14 { + status = "okay"; + compatible = "goodix,gt1x"; + reg = <0x14>; + pinctrl-names = "default"; + pinctrl-0 = <&touch_gpio>; + goodix,rst-gpio = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>; + goodix,irq-gpio = <&gpio0 RK_PC5 IRQ_TYPE_LEVEL_LOW>; + power-supply = <&vcc3v3_lcd_n>; + }; +}; + +&i2c1 { + status = "okay"; + + rk806: pmic@23 { + compatible = "rockchip,rk806"; + reg = <0x23>; + + interrupt-parent = <&gpio0>; + interrupts = <6 IRQ_TYPE_LEVEL_LOW>; + + pinctrl-names = "default", "pmic-power-off"; + pinctrl-0 = <&pmic_pins>, <&rk806_dvs1_null>, <&rk806_dvs2_null>, <&rk806_dvs3_null>; + pinctrl-1 = <&rk806_dvs1_pwrdn>; + + /* 2800mv-3500mv */ + low_voltage_threshold = <3000>; + /* 2700mv-3400mv */ + shutdown_voltage_threshold = <2700>; + /* 140 160 */ + shutdown_temperture_threshold = <160>; + hotdie_temperture_threshold = <115>; + + /* PWRON_ON_TIME: 0:500mS; 1:20mS */ + pwron-on-time-500ms; + + /* 0: restart PMU; + * 1: reset all the power off reset registers, + * forcing the state to switch to ACTIVE mode; + * 2: Reset all the power off reset registers, + * forcing the state to switch to ACTIVE mode, + * and simultaneously pull down the RESETB PIN for 5mS before releasing + */ + pmic-reset-func = <1>; + /* buck5 external feedback resister disable */ + buck5-feedback-disable; + + vcc1-supply = <&vcc_5v0_sys_s5>; + vcc2-supply = <&vcc_5v0_sys_s5>; + vcc3-supply = <&vcc_5v0_sys_s5>; + vcc4-supply = <&vcc_5v0_sys_s5>; + vcc5-supply = <&vcc_5v0_sys_s5>; + vcc6-supply = <&vcc_5v0_sys_s5>; + vcc7-supply = <&vcc_5v0_sys_s5>; + vcc8-supply = <&vcc_5v0_sys_s5>; + vcc9-supply = <&vcc_5v0_sys_s5>; + vcc10-supply = <&vcc_5v0_sys_s5>; + vcc11-supply = <&vcc_2v0_pldo_s3>; + vcc12-supply = <&vcc_5v0_sys_s5>; + vcc13-supply = <&vcc_1v1_nldo_s3>; + vcc14-supply = <&vcc_1v1_nldo_s3>; + vcca-supply = <&vcc_5v0_sys_s5>; + + pwrkey { + status = "okay"; + }; + + pinctrl_rk806: pinctrl_rk806 { + gpio-controller; + #gpio-cells = <2>; + + rk806_dvs1_null: rk806_dvs1_null { + pins = "gpio_pwrctrl2"; + function = "pin_fun0"; + }; + + rk806_dvs1_slp: rk806_dvs1_slp { + pins = "gpio_pwrctrl1"; + function = "pin_fun1"; + }; + + rk806_dvs1_pwrdn: rk806_dvs1_pwrdn { + pins = "gpio_pwrctrl1"; + function = "pin_fun2"; + }; + + rk806_dvs1_rst: rk806_dvs1_rst { + pins = "gpio_pwrctrl1"; + function = "pin_fun3"; + }; + + rk806_dvs2_null: rk806_dvs2_null { + pins = "gpio_pwrctrl2"; + function = "pin_fun0"; + }; + + rk806_dvs2_slp: rk806_dvs2_slp { + pins = "gpio_pwrctrl2"; + function = "pin_fun1"; + }; + + rk806_dvs2_pwrdn: rk806_dvs2_pwrdn { + pins = "gpio_pwrctrl2"; + function = "pin_fun2"; + }; + + rk806_dvs2_rst: rk806_dvs2_rst { + pins = "gpio_pwrctrl2"; + function = "pin_fun3"; + }; + + rk806_dvs2_dvs: rk806_dvs2_dvs { + pins = "gpio_pwrctrl2"; + function = "pin_fun4"; + }; + + rk806_dvs2_gpio: rk806_dvs2_gpio { + pins = "gpio_pwrctrl2"; + function = "pin_fun5"; + }; + + rk806_dvs3_null: rk806_dvs3_null { + pins = "gpio_pwrctrl3"; + function = "pin_fun0"; + }; + + rk806_dvs3_slp: rk806_dvs3_slp { + pins = "gpio_pwrctrl3"; + function = "pin_fun1"; + }; + + rk806_dvs3_pwrdn: rk806_dvs3_pwrdn { + pins = "gpio_pwrctrl3"; + function = "pin_fun2"; + }; + + rk806_dvs3_rst: rk806_dvs3_rst { + pins = "gpio_pwrctrl3"; + function = "pin_fun3"; + }; + + rk806_dvs3_dvs: rk806_dvs3_dvs { + pins = "gpio_pwrctrl3"; + function = "pin_fun4"; + }; + + rk806_dvs3_gpio: rk806_dvs3_gpio { + pins = "gpio_pwrctrl3"; + function = "pin_fun5"; + }; + }; + + regulators { + vdd_cpu_big_s0: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-init-microvolt = <850000>; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <950000>; + regulator-ramp-delay = <12500>; + regulator-name = "vdd_cpu_big_s0"; + regulator-enable-ramp-delay = <400>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_npu_s0: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <950000>; + regulator-ramp-delay = <12500>; + regulator-name = "vdd_npu_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_cpu_lit_s0: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-init-microvolt = <850000>; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <950000>; + regulator-ramp-delay = <12500>; + regulator-name = "vdd_cpu_lit_s0"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <750000>; + }; + }; + + vcc_3v3_s3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_3v3_s3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vdd_gpu_s0: DCDC_REG5 { + regulator-boot-on; + regulator-init-microvolt = <750000>; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <900000>; + regulator-ramp-delay = <12500>; + regulator-name = "vdd_gpu_s0"; + regulator-enable-ramp-delay = <400>; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <850000>; + }; + }; + + vddq_ddr_s0: DCDC_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vddq_ddr_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_logic_s0: vdd_log_mem_s0: DCDC_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-init-microvolt = <750000>; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <800000>; + regulator-name = "vdd_logic_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8_s3: DCDC_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8_s3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd2_ddr_s3: DCDC_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vdd2_ddr_s3"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vdd_ddr_s0: DCDC_REG10 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <550000>; + regulator-max-microvolt = <1200000>; + regulator-name = "vdd_ddr_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca_1v8_s0: PLDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca_1v8_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8_cam: PLDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8_cam"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda_1v2_s0: PLDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-name = "vdda_1v2_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcca3v3_codec: PLDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcca3v3_codec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd_s0: PLDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd_s0"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcca1v8_pldo6_s3: PLDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pldo6_s3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdd_0v75_s3: NLDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <750000>; + regulator-name = "vdd_0v75_s3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <750000>; + }; + }; + + vdda_ddr_pll_s0: NLDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <850000>; + regulator-max-microvolt = <850000>; + regulator-name = "vdda_ddr_pll_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v75_hdmi_s0: NLDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <750000>; + regulator-name = "vdda0v75_hdmi_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda_0v85_s0: NLDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <850000>; + regulator-max-microvolt = <850000>; + regulator-name = "vdda_0v85_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda_0v75_s0: NLDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <750000>; + regulator-name = "vdda_0v75_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; +}; + +&i2c7 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c7m2_xfer>; + + hym8563: hym8563@51 { + compatible = "haoyu,hym8563"; + reg = <0x51>; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "hym8563"; + pinctrl-names = "default"; + pinctrl-0 = <&rtc_int>; + + interrupt-parent = <&gpio0>; + interrupts = ; + wakeup-source; + }; +}; + +&iep { + status = "okay"; +}; + +&iep_mmu { + status = "okay"; +}; + +&jpegd { + status = "okay"; +}; + +&jpege { + status = "okay"; +}; + +&jpeg_mmu { + status = "okay"; +}; + +&mdio0 { + rgmii_phy0: phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x1>; + clocks = <&cru REFCLKO25M_GMAC0_OUT>; + }; +}; + +&mdio1 { + rgmii_phy1: phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0x1>; + clocks = <&cru REFCLKO25M_GMAC1_OUT>; + }; +}; + +&mipidcphy0 { + status = "okay"; +}; + +&mpp_srv { + status = "okay"; +}; + +&pcie0 { + rockchip,skip-scan-in-resume; + reset-gpios = <&gpio4 RK_PC7 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&vcc_3v3_pcie>; + status = "okay"; +}; + +&pinctrl { + hym8563 { + rtc_int: rtc-int { + rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + gpio-leds { + leds_gpio: leds-gpio { + rockchip,pins = <3 16 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pcie { + pcie_reset_gpio: pcie-reset-gpio { + rockchip,pins = <4 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + pcie_wake_gpio: pcie-wake-gpio { + rockchip,pins = <0 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + pcie_pwren: pcie-pwren { + rockchip,pins = <2 RK_PC7 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sd { + sd_pwren: sd-pwren { + rockchip,pins = <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + touch { + touch_gpio: touch-gpio { + rockchip,pins = + <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>, + <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb { + usb_host_pwren: usb-host-pwren { + rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + // usb_otg_pwren: usb-otg-pwren { + // rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + // }; + }; + + wireless-bluetooth { + uart7_gpios: uart7-gpios { + rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + bt_host_wake_irq: bt-host-wake-irq { + rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + bt_reset_gpio: bt-reset-gpio { + rockchip,pins = <2 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + bt_wake_gpio: bt-wake-gpio { + rockchip,pins = <2 RK_PC6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + wireless-wlan { + wifi_host_wake_irq: wifi-host-wake-irq { + rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + wifi_poweren_gpio: wifi-poweren-gpio { + rockchip,pins = <2 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; +}; + +&pwm1_6ch_1 { + status = "okay"; + pinctrl-0 = <&pwm1m0_ch1>; +}; + +&pwm1_6ch_4 { + status = "okay"; + pinctrl-0 = <&pwm1m0_ch4>; +}; + +&rga2_core0 { + status = "okay"; +}; + +&rga2_core0_mmu { + status = "okay"; +}; + +&rga2_core1 { + status = "okay"; +}; + +&rga2_core1_mmu { + status = "okay"; +}; + +&rknpu { + rknpu-supply = <&vdd_npu_s0>; + status = "okay"; +}; + +&rknpu_mmu { + status = "okay"; +}; + +&rkvenc_ccu { + status = "okay"; +}; + +&rkvenc0 { + status = "okay"; +}; + +&rkvenc0_mmu { + status = "okay"; +}; + +&rkvenc1 { + status = "okay"; +}; + +&rkvenc1_mmu { + status = "okay"; + +}; + +&rkvdec { + status = "okay"; +}; + +&rkvdec_mmu { + status = "okay"; +}; + +&route_dsi { + status = "disabled"; + connect = <&vp1_out_dsi>; +}; + +&route_hdmi { + status = "okay"; + connect = <&vp0_out_hdmi>; +}; + +&sai2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&sai2m1_lrck + &sai2m1_sclk + &sai2m1_sdi + &sai2m1_sdo>; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcca_1v8_s0>; +}; + +&sdhci { + bus-width = <8>; + no-sdio; + no-sd; + non-removable; + max-frequency = <200000000>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + full-pwr-cycle-in-suspend; + status = "okay"; +}; + +&sdio { + max-frequency = <150000000>; + no-sd; + no-mmc; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc1m1_bus4 &sdmmc1m1_clk &sdmmc1m1_cmd>; + sd-uhs-sdr104; + status = "okay"; +}; + +&sdmmc { + supports-sd; + status = "okay"; +}; + +&rockchip_suspend { + status = "okay"; +}; + +&uart7 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart7m0_xfer &uart7m0_ctsn>; +}; + +&u2phy0 { + status = "okay"; +}; + +&u2phy1 { + status = "okay"; +}; + +&u2phy0_otg { + status = "okay"; +}; + +&u2phy1_otg { + phy-supply = <&vcc5v0_usb30_host1>; + status = "okay"; +}; + +&ufs { + reset-gpios = <&gpio4 RK_PD0 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&usbdp_phy { + status = "okay"; +}; + +&usbdp_phy_dp { + status = "disabled"; +}; + +&usbdp_phy_u3 { + status = "okay"; +}; + +&usb_drd0_dwc3 { + dr_mode = "peripheral"; + extcon = <&u2phy0>; + status = "okay"; +}; + +&usb_drd1_dwc3 { + dr_mode = "host"; + // extcon = <&u2phy1>; + status = "okay"; +}; + +&vdpp { + status = "okay"; +}; + +&vop { + status = "okay"; + vop-supply = <&vdd_logic_s0>; +}; + +&vop_mmu { + status = "okay"; +}; + +&vp2 { + assigned-clocks = <&cru DCLK_VP2_SRC>; + assigned-clock-parents = <&cru PLL_VPLL>; +}; From 36ea4109ba79b5c98aa08e7a5e307e4b0b961955 Mon Sep 17 00:00:00 2001 From: Zefa Chen Date: Tue, 10 Jun 2025 18:08:53 +0800 Subject: [PATCH 32/49] media: rockchip: vicap: support reserved mem attach dma_buf with iommu Change-Id: Ibd8d54ddda18d2cd5ac7d3d38808fd52a0048f43 Signed-off-by: Zefa Chen --- drivers/media/platform/rockchip/cif/capture.c | 1 + drivers/media/platform/rockchip/cif/common.c | 65 ++++++++++++++++--- drivers/media/platform/rockchip/cif/common.h | 1 + drivers/media/platform/rockchip/cif/hw.h | 2 + .../media/platform/rockchip/cif/subdev-itf.c | 5 +- 5 files changed, 63 insertions(+), 11 deletions(-) diff --git a/drivers/media/platform/rockchip/cif/capture.c b/drivers/media/platform/rockchip/cif/capture.c index dd4cf893139e..ca3db74adb8e 100644 --- a/drivers/media/platform/rockchip/cif/capture.c +++ b/drivers/media/platform/rockchip/cif/capture.c @@ -6171,6 +6171,7 @@ void rkcif_free_rx_buf(struct rkcif_stream *stream, int buf_num) if (buf->dbufs.is_init) v4l2_subdev_call(sd, core, ioctl, RKISP_VICAP_CMD_RX_BUFFER_FREE, &buf->dbufs); + rkcif_free_reserved_mem_buf(dev, buf); memset(buf, 0, sizeof(*buf)); buf->dummy.is_free = true; } diff --git a/drivers/media/platform/rockchip/cif/common.c b/drivers/media/platform/rockchip/cif/common.c index 9f5cdb3e037d..61ec92440b8b 100644 --- a/drivers/media/platform/rockchip/cif/common.c +++ b/drivers/media/platform/rockchip/cif/common.c @@ -343,17 +343,42 @@ int rkcif_alloc_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffe { struct rkcif_dummy_buffer *dummy = &buf->dummy; u32 reserved_mem = 0; + struct dma_buf_attachment *dba; + struct sg_table *sgt; + dma_addr_t dma; + int ret = 0; + u32 dma_addr = 0; if (dev->pre_buf_num) reserved_mem = SHARED_MEM_RESERVED_HEAD_SIZE; - dummy->dma_addr = reserved_mem + dev->resmem_pa + dummy->size * buf->buf_idx; - if (dummy->dma_addr + dummy->size > dev->resmem_pa + dev->resmem_size) + dma_addr = reserved_mem + dev->resmem_pa + dummy->size * buf->buf_idx; + if (dma_addr + dummy->size > dev->resmem_pa + dev->resmem_size) return -EINVAL; - buf->dbufs.dma = dummy->dma_addr; + buf->dbufs.dma = dma_addr; buf->dbufs.is_resmem = true; - buf->shmem.shm_start = dummy->dma_addr; + buf->shmem.shm_start = dma_addr; buf->shmem.shm_size = dummy->size; dummy->dbuf = rkcif_shm_alloc(&buf->shmem); + buf->dbufs.dbuf = dummy->dbuf; + if (dev->hw_dev->iommu_en) { + dba = dma_buf_attach(dummy->dbuf, dev->hw_dev->dev); + if (IS_ERR(dba)) { + ret = PTR_ERR(dba); + goto err_alloc; + } + dummy->dba = dba; + sgt = dma_buf_map_attachment(dba, DMA_BIDIRECTIONAL); + if (IS_ERR(sgt)) { + ret = PTR_ERR(sgt); + goto err_alloc; + } + dummy->sgt = sgt; + dma = sg_dma_address(sgt->sgl); + get_dma_buf(dummy->dbuf); + dummy->dma_addr = dma; + } else { + dummy->dma_addr = dma_addr; + } if (dummy->is_need_vaddr) { struct iosys_map map; @@ -361,6 +386,11 @@ int rkcif_alloc_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffe dummy->vaddr = map.vaddr; } return 0; +err_alloc: + v4l2_info(&dev->v4l2_dev, + "can't match dma_buf 0x%x with iommu\n", + (u32)dummy->dma_addr); + return ret; } void rkcif_free_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffer *buf) @@ -396,13 +426,30 @@ void rkcif_free_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffe if (buf->dbufs.is_init) v4l2_subdev_call(sd, core, ioctl, RKISP_VICAP_CMD_RX_BUFFER_FREE, &buf->dbufs); + if (dev->hw_dev->iommu_en) { + if (dummy->dba) { + if (dummy->sgt) { + dma_buf_unmap_attachment(dummy->dba, dummy->sgt, + DMA_BIDIRECTIONAL); + dummy->sgt = NULL; + } + dma_buf_detach(dummy->dbuf, dummy->dba); + dma_buf_put(dummy->dbuf); + dummy->dba = NULL; + } + } if (dummy->is_need_vaddr) dummy->dbuf->ops->vunmap(dummy->dbuf, NULL); -#ifdef CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_ISP - free_reserved_area(phys_to_virt(buf->shmem.shm_start), - phys_to_virt(buf->shmem.shm_start + buf->shmem.shm_size), - -1, "rkisp_thunderboot"); -#endif + dma_buf_put(dummy->dbuf); buf->dummy.is_free = true; } +void rkcif_free_reserved_mem(u32 start, u32 size) +{ +#ifdef CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_ISP + free_reserved_area(phys_to_virt(start), + phys_to_virt(start + size), + -1, "rkisp_thunderboot"); +#endif +} + diff --git a/drivers/media/platform/rockchip/cif/common.h b/drivers/media/platform/rockchip/cif/common.h index 7bdfe4f40223..8745df4508f8 100644 --- a/drivers/media/platform/rockchip/cif/common.h +++ b/drivers/media/platform/rockchip/cif/common.h @@ -25,6 +25,7 @@ void rkcif_free_common_dummy_buf(struct rkcif_device *dev, struct rkcif_dummy_bu int rkcif_alloc_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffer *buf); void rkcif_free_reserved_mem_buf(struct rkcif_device *dev, struct rkcif_rx_buffer *buf); +void rkcif_free_reserved_mem(u32 start, u32 size); #endif /* _RKCIF_COMMON_H */ diff --git a/drivers/media/platform/rockchip/cif/hw.h b/drivers/media/platform/rockchip/cif/hw.h index 537dc154b329..c0ecbf2ab78c 100644 --- a/drivers/media/platform/rockchip/cif/hw.h +++ b/drivers/media/platform/rockchip/cif/hw.h @@ -81,6 +81,8 @@ struct rkcif_dummy_buffer { struct vb2_queue vb2_queue; struct list_head list; struct dma_buf *dbuf; + struct dma_buf_attachment *dba; + struct sg_table *sgt; dma_addr_t dma_addr; struct page **pages; void *mem_priv; diff --git a/drivers/media/platform/rockchip/cif/subdev-itf.c b/drivers/media/platform/rockchip/cif/subdev-itf.c index 26a6423d88f2..d47d38be3b5f 100644 --- a/drivers/media/platform/rockchip/cif/subdev-itf.c +++ b/drivers/media/platform/rockchip/cif/subdev-itf.c @@ -98,6 +98,7 @@ static void sditf_buffree_work(struct work_struct *work) if (rx_buf) { list_del(&rx_buf->list_free); rkcif_free_reserved_mem_buf(priv->cif_dev, rx_buf); + rkcif_free_reserved_mem(rx_buf->shmem.shm_start, rx_buf->shmem.shm_size); memset(rx_buf, 0, sizeof(*rx_buf)); rx_buf->dummy.is_free = true; } @@ -1444,7 +1445,7 @@ static int sditf_s_rx_buffer(struct v4l2_subdev *sd, if (cif_dev->is_thunderboot || cif_dev->is_rtt_suspend || cif_dev->is_aov_reserved) - dma_sync_single_for_device(cif_dev->dev, + dma_sync_single_for_device(cif_dev->hw_dev->dev, rx_buf->dummy.dma_addr + rx_buf->dummy.size - stream->pixm.plane_fmt[0].bytesperline * 3, stream->pixm.plane_fmt[0].bytesperline * 3, @@ -1453,6 +1454,7 @@ static int sditf_s_rx_buffer(struct v4l2_subdev *sd, cif_dev->hw_dev->mem_ops->prepare(rx_buf->dummy.mem_priv); } } + spin_unlock_irqrestore(&stream->vbq_lock, flags); if (dbufs->is_switch && dbufs->type == BUF_SHORT) { if (stream->is_in_vblank || !stream->dma_en) { @@ -1465,7 +1467,6 @@ static int sditf_s_rx_buffer(struct v4l2_subdev *sd, v4l2_dbg(3, rkcif_debug, &cif_dev->v4l2_dev, "switch to online mode\n"); } - spin_unlock_irqrestore(&stream->vbq_lock, flags); spin_lock_irqsave(&stream->cifdev->stream_spinlock, flags); stream->is_finish_single_cap = true; From e784010e9354c0cd1c66eee40bb772a8291962e8 Mon Sep 17 00:00:00 2001 From: Guochun Huang Date: Thu, 12 Jun 2025 17:37:56 +0800 Subject: [PATCH 33/49] arm64: dts: rockchip: rk3588/rk3576/rk3568-evb: delay init-delay-ms before panel initail code send Change-Id: Ib7ca5868583277dd8dba7f3c328fac77c8db3042 Signed-off-by: Guochun Huang --- arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi | 2 ++ arch/arm64/boot/dts/rockchip/rk3576-evb.dtsi | 1 + arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi | 2 ++ 3 files changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi index b7b36b5b9bab..f2da0b2e62ea 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb.dtsi @@ -396,6 +396,7 @@ reset-delay-ms = <60>; enable-delay-ms = <60>; prepare-delay-ms = <60>; + init-delay-ms = <10>; unprepare-delay-ms = <60>; disable-delay-ms = <60>; width-mm = <68>; @@ -730,6 +731,7 @@ reset-delay-ms = <60>; enable-delay-ms = <60>; prepare-delay-ms = <60>; + init-delay-ms = <10>; unprepare-delay-ms = <60>; disable-delay-ms = <60>; width-mm = <68>; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-evb.dtsi index 60fec7dc8f08..3ae8e1ddb9c2 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576-evb.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3576-evb.dtsi @@ -270,6 +270,7 @@ reset-delay-ms = <10>; enable-delay-ms = <10>; prepare-delay-ms = <10>; + init-delay-ms = <10>; unprepare-delay-ms = <10>; disable-delay-ms = <60>; width-mm = <68>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi index 925445f42981..a2d471821611 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi @@ -296,6 +296,7 @@ reset-delay-ms = <10>; enable-delay-ms = <10>; prepare-delay-ms = <10>; + init-delay-ms = <10>; unprepare-delay-ms = <10>; disable-delay-ms = <60>; width-mm = <68>; @@ -630,6 +631,7 @@ reset-delay-ms = <10>; enable-delay-ms = <10>; prepare-delay-ms = <10>; + init-delay-ms = <10>; unprepare-delay-ms = <10>; disable-delay-ms = <10>; width-mm = <68>; From 386638e520388408431eb112763bb3073d24ea73 Mon Sep 17 00:00:00 2001 From: Elaine Zhang Date: Tue, 17 Jun 2025 10:29:25 +0800 Subject: [PATCH 34/49] clk: rockchip: rv1126b: fix clk_cpll_div10 parent clk v0: gpll ----| |--DIV--clk_cpll_div10 cpll ----| v1: gpll --------------| |--DIV--clk_cpll_div10 clk_asip_pll_src --| Fixes: d804f2398824 ("clk: rockchip: rv1126b: fix clk_cpll_div10 parent clk") Change-Id: I94afa93a33db16644ddab531fba1260b3bf9c48a Signed-off-by: Elaine Zhang --- drivers/clk/rockchip/clk-rv1126b.c | 14 +++++++++----- include/dt-bindings/clock/rockchip,rv1126b-cru.h | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/clk/rockchip/clk-rv1126b.c b/drivers/clk/rockchip/clk-rv1126b.c index 96b939b63a34..ed7cac4c2953 100644 --- a/drivers/clk/rockchip/clk-rv1126b.c +++ b/drivers/clk/rockchip/clk-rv1126b.c @@ -144,7 +144,7 @@ PNAME(clk_timer3_parents_p) = { "clk_timer_root", "mclk_asrc0", "mclk_asrc1" }; PNAME(clk_timer4_parents_p) = { "clk_timer_root", "mclk_asrc2", "mclk_asrc3" }; PNAME(clk_macphy_p) = { "xin24m", "clk_cpll_div20" }; PNAME(mux_ddrphy_p) = { "dpll", "aclk_sysmem" }; -PNAME(clk_cpll_div10_p) = { "gpll", "clk_aisp_pll" }; +PNAME(clk_cpll_div10_p) = { "gpll", "clk_aisp_pll_src" }; static struct rockchip_pll_clock rv1126b_pll_clks[] __initdata = { [gpll] = PLL(pll_rk3328, PLL_GPLL, "gpll", mux_pll_p, @@ -1016,9 +1016,11 @@ static struct rockchip_clk_branch rv1126b_clk_branches[] __initdata = { }; static struct rockchip_clk_branch rv1126b_clk_cpll_div10_v0[] __initdata = { - COMPOSITE(CLK_AISP_PLL, "clk_aisp_pll", mux_gpll_aupll_cpll_p, 0, - RV1126B_CLKSEL_CON(62), 4, 2, MFLAGS, 0, 3, DFLAGS, + COMPOSITE_NODIV(CLK_AISP_PLL_SRC, "clk_aisp_pll_src", mux_gpll_aupll_cpll_p, 0, + RV1126B_CLKSEL_CON(62), 4, 2, MFLAGS, RV1126B_CLKGATE_CON(5), 4, GFLAGS), + DIV(CLK_AISP_PLL, "clk_aisp_pll", "clk_aisp_pll_src", 0, + RV1126B_CLKSEL_CON(62), 0, 3, DFLAGS), COMPOSITE(CLK_CPLL_DIV10, "clk_cpll_div10", mux_gpll_cpll_p, 0, RV1126B_CLKSEL_CON(1), 15, 1, MFLAGS, 5, 5, DFLAGS, @@ -1026,9 +1028,11 @@ static struct rockchip_clk_branch rv1126b_clk_cpll_div10_v0[] __initdata = { }; static struct rockchip_clk_branch rv1126b_clk_cpll_div10[] __initdata = { - COMPOSITE(CLK_AISP_PLL, "clk_aisp_pll", mux_gpll_aupll_cpll_p, 0, - RV1126B_CLKSEL_CON(62), 4, 2, MFLAGS, 0, 3, DFLAGS, + COMPOSITE_NODIV(CLK_AISP_PLL_SRC, "clk_aisp_pll_src", mux_gpll_aupll_cpll_p, 0, + RV1126B_CLKSEL_CON(62), 4, 2, MFLAGS, RV1126B_CLKGATE_CON(5), 4, GFLAGS), + DIV(CLK_AISP_PLL, "clk_aisp_pll", "clk_aisp_pll_src", 0, + RV1126B_CLKSEL_CON(62), 0, 3, DFLAGS), COMPOSITE(CLK_CPLL_DIV10, "clk_cpll_div10", clk_cpll_div10_p, 0, RV1126B_CLKSEL_CON(1), 15, 1, MFLAGS, 5, 5, DFLAGS, diff --git a/include/dt-bindings/clock/rockchip,rv1126b-cru.h b/include/dt-bindings/clock/rockchip,rv1126b-cru.h index c10caa9d25e8..09aa04325b51 100644 --- a/include/dt-bindings/clock/rockchip,rv1126b-cru.h +++ b/include/dt-bindings/clock/rockchip,rv1126b-cru.h @@ -371,6 +371,7 @@ #define CLK_MACPHY 361 #define HCLK_RKRNG_NS 362 #define HCLK_RKRNG_S_NS 363 +#define CLK_AISP_PLL_SRC 364 /* secure clks */ #define CLK_USER_OTPC_S 400 From 7a1e3b7f13d91b7602e78b9ba7bbb290647fb601 Mon Sep 17 00:00:00 2001 From: Elaine Zhang Date: Tue, 17 Jun 2025 10:32:01 +0800 Subject: [PATCH 35/49] arm64: dts: rockchip: rv1126b: Fix clk_aisp_pll_src init freq Fixes: dc6b7b171521 ("arm64: dts: rockchip: rv1126b: add clk_cpll_div10 assigned clk rate") Change-Id: I8f90672f97d8fadf2dd548e6451b926efd53569e Signed-off-by: Elaine Zhang --- arch/arm64/boot/dts/rockchip/rv1126b.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/rockchip/rv1126b.dtsi b/arch/arm64/boot/dts/rockchip/rv1126b.dtsi index a75da64b8ba6..866f4238d7ea 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b.dtsi +++ b/arch/arm64/boot/dts/rockchip/rv1126b.dtsi @@ -940,7 +940,7 @@ <&cru CLK_UART_FRAC0>, <&cru CLK_UART_FRAC1>, <&cru CLK_CM_FRAC0>, <&cru CLK_CM_FRAC1>, <&cru CLK_CM_FRAC2>, <&cru CLK_AUDIO_FRAC0>, - <&cru CLK_AUDIO_FRAC1>, <&cru CLK_AISP_PLL>, + <&cru CLK_AUDIO_FRAC1>, <&cru CLK_AISP_PLL_SRC>, <&cru CLK_CPLL_DIV10>; assigned-clock-rates = <1188000000>, <1000000000>, @@ -951,7 +951,7 @@ <96000000>, <128000000>, <18432000>, <500000000>, <32768000>, <45158400>, - <49152000>, <393216000>, + <49152000>, <786432000>, <98304000>; }; From 449f748d6eb8ed42b597b28f8bdb3af110641ba3 Mon Sep 17 00:00:00 2001 From: Luo Wei Date: Mon, 14 Apr 2025 17:30:01 +0800 Subject: [PATCH 36/49] misc: vehicle: add vehicle drivers with protobuf for vhal it support adc,gpio,spi etc interface Signed-off-by: Luo Wei Signed-off-by: Tom Song Change-Id: I5006c0eaa4a72eaa5ba3319c0ac1a2a6f960ede1 --- drivers/misc/Kconfig | 1 + drivers/misc/Makefile | 1 + drivers/misc/vehicle/Kconfig | 60 + drivers/misc/vehicle/Makefile | 27 + drivers/misc/vehicle/core.h | 374 ++++ drivers/misc/vehicle/nanopb-c/Makefile | 6 + drivers/misc/vehicle/nanopb-c/pb.h | 643 +++++++ drivers/misc/vehicle/nanopb-c/pb_common.c | 108 ++ drivers/misc/vehicle/nanopb-c/pb_common.h | 68 + drivers/misc/vehicle/nanopb-c/pb_decode.c | 1503 +++++++++++++++++ drivers/misc/vehicle/nanopb-c/pb_decode.h | 201 +++ drivers/misc/vehicle/nanopb-c/pb_encode.c | 889 ++++++++++ drivers/misc/vehicle/nanopb-c/pb_encode.h | 204 +++ drivers/misc/vehicle/nanopb-c/pb_system.h | 27 + drivers/misc/vehicle/vehicle-adc.c | 274 +++ drivers/misc/vehicle/vehicle-gpio.c | 230 +++ drivers/misc/vehicle/vehicle-spi.c | 413 +++++ drivers/misc/vehicle/vehicle_core.c | 418 +++++ drivers/misc/vehicle/vehicle_core.h | 388 +++++ drivers/misc/vehicle/vehicle_dummy_hw.c | 916 ++++++++++ .../misc/vehicle/vehicle_protocol_callback.c | 336 ++++ .../misc/vehicle/vehicle_protocol_callback.h | 79 + drivers/misc/vehicle/vehiclehalproto.pb.c | 166 ++ drivers/misc/vehicle/vehiclehalproto.pb.h | 191 +++ 24 files changed, 7523 insertions(+) create mode 100644 drivers/misc/vehicle/Kconfig create mode 100644 drivers/misc/vehicle/Makefile create mode 100644 drivers/misc/vehicle/core.h create mode 100644 drivers/misc/vehicle/nanopb-c/Makefile create mode 100644 drivers/misc/vehicle/nanopb-c/pb.h create mode 100644 drivers/misc/vehicle/nanopb-c/pb_common.c create mode 100644 drivers/misc/vehicle/nanopb-c/pb_common.h create mode 100644 drivers/misc/vehicle/nanopb-c/pb_decode.c create mode 100644 drivers/misc/vehicle/nanopb-c/pb_decode.h create mode 100644 drivers/misc/vehicle/nanopb-c/pb_encode.c create mode 100644 drivers/misc/vehicle/nanopb-c/pb_encode.h create mode 100644 drivers/misc/vehicle/nanopb-c/pb_system.h create mode 100644 drivers/misc/vehicle/vehicle-adc.c create mode 100644 drivers/misc/vehicle/vehicle-gpio.c create mode 100644 drivers/misc/vehicle/vehicle-spi.c create mode 100644 drivers/misc/vehicle/vehicle_core.c create mode 100644 drivers/misc/vehicle/vehicle_core.h create mode 100644 drivers/misc/vehicle/vehicle_dummy_hw.c create mode 100644 drivers/misc/vehicle/vehicle_protocol_callback.c create mode 100644 drivers/misc/vehicle/vehicle_protocol_callback.h create mode 100644 drivers/misc/vehicle/vehiclehalproto.pb.c create mode 100644 drivers/misc/vehicle/vehiclehalproto.pb.h diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 0c5c5f6d54c1..f34f8fe2f36a 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -14,6 +14,7 @@ config RK803 Driver for RK803 which is used for driving porjector and IR flood LED. source "drivers/misc/rockchip/Kconfig" +source "drivers/misc/vehicle/Kconfig" config LT7911D_FB_NOTIFIER tristate "Lontium LT7911D FB Notifier" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index 0d1eab280bab..d12571822641 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -6,6 +6,7 @@ obj-y += rk628/ obj-$(CONFIG_RK803) += rk803.o obj-y += rockchip/ +obj-y += vehicle/ obj-$(CONFIG_LT7911D_FB_NOTIFIER) += lt7911d-fb-notifier.o obj-$(CONFIG_IBM_ASM) += ibmasm/ obj-$(CONFIG_IBMVMC) += ibmvmc.o diff --git a/drivers/misc/vehicle/Kconfig b/drivers/misc/vehicle/Kconfig new file mode 100644 index 000000000000..64b7eba956e2 --- /dev/null +++ b/drivers/misc/vehicle/Kconfig @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# vehicle configuration +# + +menu "misc vehicle setting" + +config VEHICLE_CORE + tristate "ROCKCHIP VEHICLE core support" + help + If you say yes here you get support for the RK + vehicle core module. + +config VEHICLE_DUMMY + tristate "RK VEHICLE DUMMY support" + select EXTCON + depends on VEHICLE_CORE + help + If you say yes here you get support for the RK dummy MCU + which simulate the vehicle signal. + +config VEHICLE_ADC + tristate "VEHICLE ADC support" + depends on VEHICLE_CORE + help + If you say yes here you get support for the vehicle with adc + which simulate the vehicle signal. + +config VEHICLE_GPIO + tristate "VEHICLE GPIO support" + depends on VEHICLE_CORE + help + If you say yes here you get support for the vehicle with gpio + which simulate the vehicle signal. + +config VEHICLE_SPI + tristate "VEHICLE SPI support" + depends on VEHICLE_CORE + help + If you say yes here you get support for the vehicle with spi bus + which simulate the vehicle signal. + +config VEHICLE_SPI_PROTOCOL + tristate "VEHICLE SPI_PROTOCOL support" + depends on VEHICLE_SPI + help + If you say yes here, you can get support for vehicle spi protocol. + +config VEHICLE_GPIO_MCU_EXPANDER + tristate "VEHICLE MCU GPIO EXPANDER support" + help + If you say yes here you get support for the mcu gpio expander. + +config VEHICLE_DRIVER_OREO + bool "RK vehicle driver for android oreo" + help + If you say yes here, you can get support for vehicle protocol + in android o, otherwise it support android p. + +endmenu diff --git a/drivers/misc/vehicle/Makefile b/drivers/misc/vehicle/Makefile new file mode 100644 index 000000000000..5d245fc6964a --- /dev/null +++ b/drivers/misc/vehicle/Makefile @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for vehicle driver +# + +ccflags-y := -std=gnu99 -Wno-declaration-after-statement -DPB_SYSTEM_HEADER=\"pb_system.h\" -I$(srctree)/drivers/misc/vehicle/nanopb-c/ + +obj-$(CONFIG_VEHICLE_CORE) += vehicle-core.o +vehicle-core-objs = \ + nanopb-c/pb_common.o \ + nanopb-c/pb_decode.o \ + nanopb-c/pb_encode.o \ + vehicle_core.o \ + vehicle_protocol_callback.o \ + vehiclehalproto.pb.o + +obj-$(CONFIG_VEHICLE_DUMMY) += \ + vehicle_dummy_hw.o + +obj-$(CONFIG_VEHICLE_ADC) += \ + vehicle-adc.o + +obj-$(CONFIG_VEHICLE_GPIO) += \ + vehicle-gpio.o + +obj-$(CONFIG_VEHICLE_SPI) += \ + vehicle-spi.o diff --git a/drivers/misc/vehicle/core.h b/drivers/misc/vehicle/core.h new file mode 100644 index 000000000000..4dbac12a73e5 --- /dev/null +++ b/drivers/misc/vehicle/core.h @@ -0,0 +1,374 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * core.h -- core define for mfd display arch + * + * Copyright (c) 2023-2028 Rockchip Electronics Co., Ltd. + * + * Author: luowei + * + */ + +#ifndef __MFD_VEHICLE_CORE_H__ +#define __MFD_VEHICLE_CORE_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + + +#include "vehicle_core.h" + +/* + * if enable all the debug information, + * there will be much log. + * + * so suggest set CONFIG_LOG_BUF_SHIFT to 18 + */ +#define VEHICLE_DEBUG + + +#ifdef VEHICLE_DEBUG +#define VEHICLE_DBG(x...) pr_info(x) +#else +#define VEHICLE_DBG(x...) no_printk(x) +#endif + +#define MCU_MAX_REGS 8 +struct mcu_gpio_chip { + const char *name; + struct platform_device *pdev; + struct gpio_chip gpio_chip; + struct regmap *regmap; + struct regulator *regulator; + unsigned int ngpio; + u8 backup_regs[MCU_MAX_REGS]; + + struct gpio_desc *reset_gpio_desc; + int reset_gpio_irq; +}; + +struct vehicle; +enum vehicle_hw_type { + VEHICLE_HW_TYPE_INVALID = 0, + + VEHICLE_HW_TYPE_ADC, + VEHICLE_HW_TYPE_GPIO, + VEHICLE_HW_TYPE_I2C, + VEHICLE_HW_TYPE_SPI, + VEHICLE_HW_TYPE_UART, + VEHICLE_HW_TYPE_CHIP_MCU, + + VEHICLE_HW_TYPE_END, +}; + +/* VEHICLE is in state parking */ +#define GEAR_0 1 +/* VEHICLE is in state reverse */ +#define GEAR_1 2 +/* VEHICLE is in state neutral */ +#define GEAR_3 3 +/* VEHICLE is in state driver */ +#define GEAR_2 4 +/* no turn signal */ +#define TURN_0 0 +/* left turn signal */ +#define TURN_1 1 +/* right turn signal */ +#define TURN_2 2 + +#define POWER_REQ_STATE_ON 0 +#define POWER_REQ_STATE_SHUTDOWN_PREPARE 1 +#define POWER_REQ_STATE_CANCEL_SHUTDOWN 2 +#define POWER_REQ_STATE_FINISHED 3 + +#define POWER_REQ_PARAM_SHUTDOWN_IMMEDIATELY 1 +#define POWER_REQ_PARAM_CAN_SLEEP 2 +#define POWER_REQ_PARAM_SHUTDOWN_ONLY 3 +#define POWER_REQ_PARAM_SLEEP_IMMEDIATELY 4 +#define POWER_REQ_PARAM_HIBERNATE_IMMEDIATELY 5 +#define POWER_REQ_PARAM_CAN_HIBERNATE 6 + +/* temperature set from hardware on Android OREO and PIE uses below indexes */ +#define AC_TEMP_LEFT_INDEX 1 +#define AC_TEMP_RIGHT_INDEX 4 + +/* temperature set from APP on Android PIE uses below indexes */ +#define PIE_AC_TEMP_LEFT_INDEX 49 +#define PIE_AC_TEMP_RIGHT_INDEX 68 + + +struct vehicle_hw_data { + const char *name; + enum vehicle_hw_type vehicle_hw_type; + + int (*hw_init)(struct vehicle *vehicle); + int (*data_update)(struct vehicle *vehicle); + int (*suspend)(struct vehicle *vehicle); + int (*resume)(struct vehicle *vehicle); +}; + +struct vehicle_event_data { + u32 gear; + u32 turn; + u32 temp_right; + u32 temp_left; + u32 fan_direction; + u32 fan_speed; + u32 defrost_left; + u32 defrost_right; + u32 ac_on; + u32 auto_on; + u32 hvac_on; + u32 recirc_on; + u32 power_req_state; + u32 power_req_param; + u32 seat_temp_left; + u32 seat_temp_right; +}; + +struct vehicle_gpio { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type type; + struct device *dev; + struct platform_device *pdev; + struct vehicle *parent; + struct regmap *regmap; + int irq; + + struct gpio_desc *gear_gpio_park; + struct gpio_desc *gear_gpio_reverse; + struct gpio_desc *gear_gpio_neutral; + struct gpio_desc *gear_gpio_drive; + struct gpio_desc *gear_gpio_manual; + struct gpio_desc *turn_gpio; + struct gpio_desc *temp_right_gpio; + struct gpio_desc *temp_left_gpio; + struct gpio_desc *fan_direction_gpio; + struct gpio_desc *fan_speed_gpio; + struct gpio_desc *defrost_left_gpio; + struct gpio_desc *defrost_right_gpio; + struct gpio_desc *ac_on_gpio; + struct gpio_desc *auto_on_gpio; + struct gpio_desc *hvac_on_gpio; + struct gpio_desc *recirc_on_gpio; + struct gpio_desc *power_req_state_gpio; + struct gpio_desc *power_req_param_gpio; + struct gpio_desc *seat_temp_left_gpio; + struct gpio_desc *seat_temp_right_gpio; + + struct vehicle_hw_data *hw_data; +}; + +struct vehicle_chip_mcu { + enum vehicle_hw_type type; + struct device *dev; + struct vehicle *parent; + struct regmap *regmap; + +}; + +struct vehicle_spi { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type hw_type; + struct device *dev; + struct platform_device *pdev; + struct spi_device *spi; + int irq; + struct vehicle *parent; + struct regmap *regmap; + + struct mcu_gpio_chip *gpio_mcu; + + char *rx_buf; + int rx_len; + char *tx_buf; + int tx_len; + + struct vehicle_event_data spi_data; + struct vehicle_hw_data *hw_data; +}; + +struct vehicle_uart { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type hw_type; + struct device *dev; + struct platform_device *pdev; + struct vehicle *parent; + struct regmap *regmap; + + struct vehicle_event_data uart_data; + struct vehicle_hw_data *hw_data; +}; + +struct vehicle_dummy { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type hw_type; + struct device *dev; + struct platform_device *pdev; + struct vehicle *parent; + struct regmap *regmap; + + struct vehicle_event_data dummy_hw_data; +}; + +struct vehicle_adc { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type type; + struct device *dev; + struct platform_device *pdev; + struct vehicle *parent; + struct regmap *regmap; + int irq; + + struct iio_channel *gear_adc_chn; + struct iio_channel *turn_left_adc_chn; + struct iio_channel *turn_right_adc_chn; + struct iio_channel *temp_right_adc_chn; + struct iio_channel *temp_left_adc_chn; + struct iio_channel *fan_direction_adc_chn; + struct iio_channel *fan_speed_adc_chn; + struct iio_channel *defrost_left_adc_chn; + struct iio_channel *defrost_right_adc_chn; + struct iio_channel *ac_on_adc_chn; + struct iio_channel *auto_on_adc_chn; + struct iio_channel *hvac_on_adc_chn; + struct iio_channel *recirc_on_adc_chn; + struct iio_channel *power_req_state_adc_chn; + struct iio_channel *power_req_param_adc_chn; + struct iio_channel *seat_temp_left_adc_chn; + struct iio_channel *seat_temp_right_adc_chn; + + struct vehicle_hw_data *hw_data; +}; + +struct vehicle_i2c { + struct mutex wq_lock; + int use_delay_work; + struct delayed_work irq_work; + struct workqueue_struct *vehicle_wq; + struct delayed_work vehicle_delay_work; + + enum vehicle_hw_type hw_type; + struct device *dev; + struct platform_device *pdev; + struct vehicle *parent; + struct regmap *regmap; + + struct i2c_client *client; + struct vehicle_event_data i2c_data; + struct vehicle_hw_data *hw_data; +}; + +struct vehicle { + enum vehicle_hw_type hw_type; + struct device *dev; + struct platform_device *pdev; + + struct vehicle_event_data vehicle_data; + + struct vehicle_adc *vehicle_adc; + struct vehicle_gpio *vehicle_gpio; + struct vehicle_i2c *vehicle_i2c; + struct vehicle_spi *vehicle_spi; + struct vehicle_uart *vehicle_uart; + struct vehicle_dummy *vehicle_dummy; + struct vehicle_chip_mcu *vehicle_chip_mcu; +}; + +extern struct vehicle_hw_data vehicle_adc_data; +extern struct vehicle_hw_data vehicle_gpio_data; +extern struct vehicle_hw_data vehicle_i2c_data; +extern struct vehicle_hw_data vehicle_spi_data; +extern struct vehicle_hw_data vehicle_uart_data; +extern struct vehicle_hw_data vehicle_chip_mcu_data; + +extern struct vehicle *g_vehicle_hw; +extern void vehicle_set_property(u16 prop, u8 index, u32 value, u32 param); +extern int gpio_mcu_register(struct spi_device *spi); +extern int vehicle_spi_write_slt(struct vehicle *vehicle, const void *txbuf, size_t n); +extern int vehicle_spi_read_slt(struct vehicle *vehicle, void *rxbuf, size_t n); + +#endif diff --git a/drivers/misc/vehicle/nanopb-c/Makefile b/drivers/misc/vehicle/nanopb-c/Makefile new file mode 100644 index 000000000000..3e2725cd98e5 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile file pb_encode pb_decode pb_common +# + +ccflags-y := -Wall -W -Wno-unused-parameter -DPB_SYSTEM_HEADER=\"pb_system.h\" diff --git a/drivers/misc/vehicle/nanopb-c/pb.h b/drivers/misc/vehicle/nanopb-c/pb.h new file mode 100644 index 000000000000..d74a6a60acd0 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb.h @@ -0,0 +1,643 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* Common parts of the nanopb library. Most of these are quite low-level + * stuff. For the high-level interface, see pb_encode.h and pb_decode.h. + */ + +#ifndef PB_H_INCLUDED +#define PB_H_INCLUDED + +/***************************************************************** + * Nanopb compilation time options. You can change these here by * + * uncommenting the lines, or on the compiler command line. * + *****************************************************************/ + +/* Enable support for dynamically allocated fields */ +/* #define PB_ENABLE_MALLOC 1 */ + +/* Define this if your CPU / compiler combination does not support + * unaligned memory access to packed structures. + */ +/* #define PB_NO_PACKED_STRUCTS 1 */ + +/* Increase the number of required fields that are tracked. + * A compiler warning will tell if you need this. + */ +/* #define PB_MAX_REQUIRED_FIELDS 256 */ + +/* Add support for tag numbers > 255 and fields larger than 255 bytes. */ +/* #define PB_FIELD_16BIT 1 */ + +/* Add support for tag numbers > 65536 and fields larger than 65536 bytes. */ +/* #define PB_FIELD_32BIT 1 */ + +/* Disable support for error messages in order to save some code space. */ +/* #define PB_NO_ERRMSG 1 */ + +/* Disable support for custom streams (support only memory buffers). */ +/* #define PB_BUFFER_ONLY 1 */ + +/* Switch back to the old-style callback function signature. + * This was the default until nanopb-0.2.1. + */ +/* #define PB_OLD_CALLBACK_STYLE */ + + +/****************************************************************** + * You usually don't need to change anything below this line. * + * Feel free to look around and use the defined macros, though. * + ****************************************************************** + */ + + +/* Version of the nanopb library. Just in case you want to check it in + * your own program. + */ +#define NANOPB_VERSION (nanopb-0.3.9.1) + +/* Include all the system headers needed by nanopb. You will need the + * definitions of the following: + * - strlen, memcpy, memset functions + * - [u]int_least8_t, uint_fast8_t, [u]int_least16_t, [u]int32_t, [u]int64_t + * - size_t + * - bool + * + * If you don't have the standard header files, you can instead provide + * a custom header that defines or includes all this. In that case, + * define PB_SYSTEM_HEADER to the path of this file. + */ +#ifdef PB_SYSTEM_HEADER +#include PB_SYSTEM_HEADER +#else +#include +#include +#include +#include + +#ifdef PB_ENABLE_MALLOC +#include +#endif +#endif + +/* Macro for defining packed structures (compiler dependent). + * This just reduces memory requirements, but is not required. + */ +#if defined(PB_NO_PACKED_STRUCTS) + /* Disable struct packing */ +#define PB_PACKED_STRUCT_START +#define PB_PACKED_STRUCT_END +#define pb_packed +#elif defined(__GNUC__) || defined(__clang__) + /* For GCC and clang */ +#define PB_PACKED_STRUCT_START +#define PB_PACKED_STRUCT_END +#define pb_packed __attribute__((packed)) +#elif defined(__ICCARM__) || defined(__CC_ARM) + /* For IAR ARM and Keil MDK-ARM compilers */ +#define PB_PACKED_STRUCT_START _Pragma("pack(push, 1)") +#define PB_PACKED_STRUCT_END _Pragma("pack(pop)") +#define pb_packed +#elif defined(_MSC_VER) && (_MSC_VER >= 1500) + /* For Microsoft Visual C++ */ +#define PB_PACKED_STRUCT_START __pragma(pack(push, 1)) +#define PB_PACKED_STRUCT_END __pragma(pack(pop)) +#define pb_packed +#else + /* Unknown compiler */ +#define PB_PACKED_STRUCT_START +#define PB_PACKED_STRUCT_END +#define pb_packed +#endif + +/* Handly macro for suppressing unreferenced-parameter compiler warnings. */ +#ifndef PB_UNUSED +#define PB_UNUSED(x) ((void)(x)) +#endif + +/* Compile-time assertion, used for checking compatible compilation options. + * If this does not work properly on your compiler, use + * #define PB_NO_STATIC_ASSERT to disable it. + * + * But before doing that, check carefully the error message / place where it + * comes from to see if the error has a real cause. Unfortunately the error + * message is not always very clear to read, but you can see the reason better + * in the place where the PB_STATIC_ASSERT macro was called. + */ +#ifndef PB_NO_STATIC_ASSERT +#ifndef PB_STATIC_ASSERT +#define PB_STATIC_ASSERT(COND, MSG) \ + typedef char PB_STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND) ? 1 : -1]; +#define PB_STATIC_ASSERT_MSG(MSG, LINE, COUNTER) PB_STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) +#define PB_STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) pb_static_assertion_##MSG##LINE##COUNTER +#endif +#ifndef STATIC_ASSERT +#define STATIC_ASSERT PB_STATIC_ASSERT +#define STATIC_ASSERT_MSG PB_STATIC_ASSERT_MSG +#define STATIC_ASSERT_MSG_ PB_STATIC_ASSERT_MSG_ +#endif +#else +#define PB_STATIC_ASSERT(COND, MSG) +#endif + +/* Number of required fields to keep track of. */ +#ifndef PB_MAX_REQUIRED_FIELDS +#define PB_MAX_REQUIRED_FIELDS 64 +#endif + +#if PB_MAX_REQUIRED_FIELDS < 64 +#error You should not lower PB_MAX_REQUIRED_FIELDS from the default value (64). +#endif + +/* List of possible field types. These are used in the autogenerated code. + * Least-significant 4 bits tell the scalar type + * Most-significant 4 bits specify repeated/required/packed etc. + */ + +typedef uint_least8_t pb_type_t; + +/**** Field data types ****/ + +/* Numeric types */ +#define PB_LTYPE_VARINT 0x00 /* int32, int64, enum, bool */ +#define PB_LTYPE_UVARINT 0x01 /* uint32, uint64 */ +#define PB_LTYPE_SVARINT 0x02 /* sint32, sint64 */ +#define PB_LTYPE_FIXED32 0x03 /* fixed32, sfixed32, float */ +#define PB_LTYPE_FIXED64 0x04 /* fixed64, sfixed64, double */ + +/* Marker for last packable field type. */ +#define PB_LTYPE_LAST_PACKABLE 0x04 + +/* Byte array with pre-allocated buffer. + * data_size is the length of the allocated PB_BYTES_ARRAY structure. + */ +#define PB_LTYPE_BYTES 0x05 + +/* String with pre-allocated buffer. + * data_size is the maximum length. + */ +#define PB_LTYPE_STRING 0x06 + +/* Submessage + * submsg_fields is pointer to field descriptions + */ +#define PB_LTYPE_SUBMESSAGE 0x07 + +/* Extension pseudo-field + * The field contains a pointer to pb_extension_t + */ +#define PB_LTYPE_EXTENSION 0x08 + +/* Byte array with inline, pre-allocated byffer. + * data_size is the length of the inline, allocated buffer. + * This differs from PB_LTYPE_BYTES by defining the element as + * pb_byte_t[data_size] rather than pb_bytes_array_t. + */ +#define PB_LTYPE_FIXED_LENGTH_BYTES 0x09 + +/* Number of declared LTYPES */ +#define PB_LTYPES_COUNT 0x0A +#define PB_LTYPE_MASK 0x0F + +/**** Field repetition rules ****/ + +#define PB_HTYPE_REQUIRED 0x00 +#define PB_HTYPE_OPTIONAL 0x10 +#define PB_HTYPE_REPEATED 0x20 +#define PB_HTYPE_ONEOF 0x30 +#define PB_HTYPE_MASK 0x30 + +/**** Field allocation types ****/ + +#define PB_ATYPE_STATIC 0x00 +#define PB_ATYPE_POINTER 0x80 +#define PB_ATYPE_CALLBACK 0x40 +#define PB_ATYPE_MASK 0xC0 + +#define PB_ATYPE(x) ((x) & PB_ATYPE_MASK) +#define PB_HTYPE(x) ((x) & PB_HTYPE_MASK) +#define PB_LTYPE(x) ((x) & PB_LTYPE_MASK) + +/* Data type used for storing sizes of struct fields + * and array counts. + */ +#if defined(PB_FIELD_32BIT) +typedef uint32_t pb_size_t; +typedef int32_t pb_ssize_t; +#elif defined(PB_FIELD_16BIT) +typedef uint_least16_t pb_size_t; +typedef int_least16_t pb_ssize_t; +#else +typedef uint_least8_t pb_size_t; +typedef int_least8_t pb_ssize_t; +#endif +#define PB_SIZE_MAX ((pb_size_t)-1) + +/* Data type for storing encoded data and other byte streams. + * This typedef exists to support platforms where uint8_t does not exist. + * You can regard it as equivalent on uint8_t on other platforms. + */ +typedef uint_least8_t pb_byte_t; + +/* This structure is used in auto-generated constants + * to specify struct fields. + * You can change field sizes if you need structures + * larger than 256 bytes or field tags larger than 256. + * The compiler should complain if your .proto has such + * structures. Fix that by defining PB_FIELD_16BIT or + * PB_FIELD_32BIT. + */ +PB_PACKED_STRUCT_START +typedef struct pb_field_s pb_field_t; +struct pb_field_s { + pb_size_t tag; + pb_type_t type; + pb_size_t data_offset; /* Offset of field data, relative to previous field. */ + pb_ssize_t size_offset; /* Offset of array size or has-boolean, relative to data */ + pb_size_t data_size; /* Data size in bytes for a single item */ + pb_size_t array_size; /* Maximum number of entries in array */ + + /* Field definitions for submessage + * OR default value for all other non-array, non-callback types + * If null, then field will zeroed. + */ + const void *ptr; +} pb_packed; +PB_PACKED_STRUCT_END + +/* Make sure that the standard integer types are of the expected sizes. + * Otherwise fixed32/fixed64 fields can break. + * + * If you get errors here, it probably means that your stdint.h is not + * correct for your platform. + */ +#ifndef PB_WITHOUT_64BIT +PB_STATIC_ASSERT(sizeof(int64_t) == 2 * sizeof(int32_t), INT64_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(uint64_t) == 2 * sizeof(uint32_t), UINT64_T_WRONG_SIZE) +#endif + +/* This structure is used for 'bytes' arrays. + * It has the number of bytes in the beginning, and after that an array. + * Note that actual structs used will have a different length of bytes array. + */ +#define PB_BYTES_ARRAY_T(n) struct { pb_size_t size; pb_byte_t bytes[n]; } +#define PB_BYTES_ARRAY_T_ALLOCSIZE(n) ((size_t)n + offsetof(pb_bytes_array_t, bytes)) + +struct pb_bytes_array_s { + pb_size_t size; + pb_byte_t bytes[1]; +}; +typedef struct pb_bytes_array_s pb_bytes_array_t; + +/* This structure is used for giving the callback function. + * It is stored in the message structure and filled in by the method that + * calls pb_decode. + * + * The decoding callback will be given a limited-length stream + * If the wire type was string, the length is the length of the string. + * If the wire type was a varint/fixed32/fixed64, the length is the length + * of the actual value. + * The function may be called multiple times (especially for repeated types, + * but also otherwise if the message happens to contain the field multiple + * times.) + * + * The encoding callback will receive the actual output stream. + * It should write all the data in one call, including the field tag and + * wire type. It can write multiple fields. + * + * The callback can be null if you want to skip a field. + */ +typedef struct pb_istream_s pb_istream_t; +typedef struct pb_ostream_s pb_ostream_t; +typedef struct pb_callback_s pb_callback_t; +struct pb_callback_s { +#ifdef PB_OLD_CALLBACK_STYLE + /* Deprecated since nanopb-0.2.1 */ +union { + bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg); + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg); +} funcs; +#else + /* New function signature, which allows modifying arg contents in callback. */ +union { + bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void **arg); + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void * const *arg); +} funcs; +#endif + + /* Free arg for use by callback */ + void *arg; +}; + +/* Wire types. Library user needs these only in encoder callbacks. */ +typedef enum { + PB_WT_VARINT = 0, + PB_WT_64BIT = 1, + PB_WT_STRING = 2, + PB_WT_32BIT = 5 +} pb_wire_type_t; + +/* Structure for defining the handling of unknown/extension fields. + * Usually the pb_extension_type_t structure is automatically generated, + * while the pb_extension_t structure is created by the user. However, + * if you want to catch all unknown fields, you can also create a custom + * pb_extension_type_t with your own callback. + */ +typedef struct pb_extension_type_s pb_extension_type_t; +typedef struct pb_extension_s pb_extension_t; +struct pb_extension_type_s { + /* Called for each unknown field in the message. + * If you handle the field, read off all of its data and return true. + * If you do not handle the field, do not read anything and return true. + * If you run into an error, return false. + * Set to NULL for default handler. + */ + bool (*decode)(pb_istream_t *stream, pb_extension_t *extension, + uint32_t tag, pb_wire_type_t wire_type); + + /* Called once after all regular fields have been encoded. + * If you have something to write, do so and return true. + * If you do not have anything to write, just return true. + * If you run into an error, return false. + * Set to NULL for default handler. + */ + bool (*encode)(pb_ostream_t *stream, const pb_extension_t *extension); + + /* Free field for use by the callback. */ + const void *arg; +}; + +struct pb_extension_s { + /* Type describing the extension field. Usually you'll initialize + * this to a pointer to the automatically generated structure. + */ + const pb_extension_type_t *type; + + /* Destination for the decoded data. This must match the datatype + * of the extension field. + */ + void *dest; + + /* Pointer to the next extension handler, or NULL. + * If this extension does not match a field, the next handler is + * automatically called. + */ + pb_extension_t *next; + + /* The decoder sets this to true if the extension was found. + * Ignored for encoding. + */ + bool found; +}; + +/* Memory allocation functions to use. You can define pb_realloc and + * pb_free to custom functions if you want. + */ +#ifdef PB_ENABLE_MALLOC +#ifndef pb_realloc +#define pb_realloc(ptr, size) realloc(ptr, size) +#endif +#ifndef pb_free +#define pb_free(ptr) free(ptr) +#endif +#endif + +/* This is used to inform about need to regenerate .pb.h/.pb.c files. */ +#define PB_PROTO_HEADER_VERSION 30 + +/* These macros are used to declare pb_field_t's in the constant array. */ +/* Size of a structure member, in bytes. */ +#define pb_membersize(st, m) ((sizeof((st *)0)->m)) +/* Number of entries in an array. */ +#define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) +/* Delta from start of one member to the start of another member. */ +#define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) +/* Marks the end of the field list */ +#define PB_LAST_FIELD {0, (pb_type_t) 0, 0, 0, 0, 0, 0} + +/* Macros for filling in the data_offset field */ +/* data_offset for first field in a message */ +#define PB_DATAOFFSET_FIRST(st, m1, m2) (offsetof(st, m1)) +/* data_offset for subsequent fields */ +#define PB_DATAOFFSET_OTHER(st, m1, m2) \ + (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) +/* data offset for subsequent fields inside a union (oneof) */ +#define PB_DATAOFFSET_UNION(st, m1, m2) (PB_SIZE_MAX) +/* Choose first/other based on m1 == m2 (deprecated, remains for backwards compatibility) */ +#define PB_DATAOFFSET_CHOOSE(st, m1, m2) \ + ((int)(offsetof(st, m1) == offsetof(st, m2)) \ + ? (PB_DATAOFFSET_FIRST(st, m1, m2)) \ + : (PB_DATAOFFSET_OTHER(st, m1, m2))) + +/* Required fields are the simplest. They just have delta (padding) from + * previous field end, and the size of the field. Pointer is used for + * submessages and default values. + */ +#define PB_REQUIRED_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +/* Optional fields add the delta to the has_ variable. */ +#define PB_OPTIONAL_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ + fd, \ + pb_delta(st, has_ ## m, m), \ + pb_membersize(st, m), 0, ptr} + +#define PB_SINGULAR_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +/* Repeated fields have a _count field and also the maximum number of entries. */ +#define PB_REPEATED_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | ltype, \ + fd, \ + pb_delta(st, m ## _count, m), \ + pb_membersize(st, m[0]), \ + pb_arraysize(st, m), ptr} + +/* Allocated fields carry the size of the actual data, not the pointer */ +#define PB_REQUIRED_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_REQUIRED | ltype, \ + fd, 0, pb_membersize(st, m[0]), 0, ptr} + +/* Optional fields don't need a has_ variable, as information would be redundant */ +#define PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m[0]), 0, ptr} + +/* Same as optional fields*/ +#define PB_SINGULAR_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m[0]), 0, ptr} + +/* Repeated fields have a _count field and a pointer to array of pointers */ +#define PB_REPEATED_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_REPEATED | ltype, \ + fd, pb_delta(st, m ## _count, m), \ + pb_membersize(st, m[0]), 0, ptr} + +/* Callbacks are much like required fields except with special datatype. */ +#define PB_REQUIRED_CALLBACK(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +#define PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +#define PB_SINGULAR_CALLBACK(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +#define PB_REPEATED_CALLBACK(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +/* Optional extensions don't have the has_ field, as that would be redundant. + * Furthermore, the combination of OPTIONAL without has_ field is used + * for indicating proto3 style fields. Extensions exist in proto2 mode only, + * so they should be encoded according to proto2 rules. To avoid the conflict, + * extensions are marked as REQUIRED instead. + */ +#define PB_OPTEXT_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \ + 0, \ + 0, \ + pb_membersize(st, m), 0, ptr} + +#define PB_OPTEXT_POINTER(tag, st, m, fd, ltype, ptr) \ + PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) + +#define PB_OPTEXT_CALLBACK(tag, st, m, fd, ltype, ptr) \ + PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) + +/* The mapping from protobuf types to LTYPEs is done using these macros. */ +#define PB_LTYPE_MAP_BOOL PB_LTYPE_VARINT +#define PB_LTYPE_MAP_BYTES PB_LTYPE_BYTES +#define PB_LTYPE_MAP_DOUBLE PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_ENUM PB_LTYPE_VARINT +#define PB_LTYPE_MAP_UENUM PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_FIXED32 PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_FIXED64 PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_FLOAT PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_INT32 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_INT64 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_MESSAGE PB_LTYPE_SUBMESSAGE +#define PB_LTYPE_MAP_SFIXED32 PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_SFIXED64 PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_SINT32 PB_LTYPE_SVARINT +#define PB_LTYPE_MAP_SINT64 PB_LTYPE_SVARINT +#define PB_LTYPE_MAP_STRING PB_LTYPE_STRING +#define PB_LTYPE_MAP_UINT32 PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_UINT64 PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_EXTENSION PB_LTYPE_EXTENSION +#define PB_LTYPE_MAP_FIXED_LENGTH_BYTES PB_LTYPE_FIXED_LENGTH_BYTES + +/* This is the actual macro used in field descriptions. + * It takes these arguments: + * - Field tag number + * - Field type: BOOL, BYTES, DOUBLE, ENUM, UENUM, FIXED32, FIXED64, + * FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64 + * SINT32, SINT64, STRING, UINT32, UINT64 or EXTENSION + * - Field rules: REQUIRED, OPTIONAL or REPEATED + * - Allocation: STATIC, CALLBACK or POINTER + * - Placement: FIRST or OTHER, depending on if this is the first field in structure. + * - Message name + * - Field name + * - Previous field name (or field name again for first field) + * - Pointer to default value or submsg fields. + */ + +#define PB_FIELD(tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ + PB_ ## rules ## _ ## allocation(tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) +#define PB_FIELD2 PB_FIELD + +/* Field description for repeated static fixed count fields.*/ +#define PB_REPEATED_FIXED_COUNT(tag, type, placement, message, field, prevfield, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | PB_LTYPE_MAP_ ## type, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ + 0, \ + pb_membersize(message, field[0]), \ + pb_arraysize(message, field), ptr} + +/* Field description for oneof fields. This requires taking into account the + * union name also, that's why a separate set of macros is needed. + */ +#define PB_ONEOF_STATIC(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, u.m), \ + pb_membersize(st, u.m), 0, ptr} + +#define PB_ONEOF_POINTER(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, u.m), \ + pb_membersize(st, u.m[0]), 0, ptr} + +#define PB_ONEOF_FIELD(union_name, tag, type, rules, \ + allocation, placement, message, field, prevfield, ptr) \ + PB_ONEOF_ ## allocation(union_name, tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, union_name.field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) + +#define PB_ANONYMOUS_ONEOF_STATIC(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, m), \ + pb_membersize(st, m), 0, ptr} + +#define PB_ANONYMOUS_ONEOF_POINTER(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, m), \ + pb_membersize(st, m[0]), 0, ptr} + +#define PB_ANONYMOUS_ONEOF_FIELD(union_name, tag, type, rules, \ + allocation, placement, message, field, prevfield, ptr) \ + PB_ANONYMOUS_ONEOF_ ## allocation(union_name, tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) + +/* These macros are used for giving out error messages. + * They are mostly a debugging aid; the main error information + * is the true/false return value from functions. + * Some code space can be saved by disabling the error + * messages if not used. + * + * PB_SET_ERROR() sets the error message if none has been set yet. + * msg must be a constant string literal. + * PB_GET_ERROR() always returns a pointer to a string. + * PB_RETURN_ERROR() sets the error and returns false from current + * function. + */ +#ifdef PB_NO_ERRMSG +#define PB_SET_ERROR(stream, msg) PB_UNUSED(stream) +#define PB_GET_ERROR(stream) "(errmsg disabled)" +#else +#define PB_SET_ERROR(stream, msg) \ + (stream->errmsg = (stream)->errmsg ? (stream)->errmsg : (msg)) +#define PB_GET_ERROR(stream) ((stream)->errmsg ? (stream)->errmsg : "(none)") +#endif + +#define PB_RETURN_ERROR(stream, msg) return (PB_SET_ERROR(stream, msg), false) + +#endif diff --git a/drivers/misc/vehicle/nanopb-c/pb_common.c b/drivers/misc/vehicle/nanopb-c/pb_common.c new file mode 100644 index 000000000000..51555d4f8f90 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_common.c @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_common.c: Common support functions for pb_encode.c and pb_decode.c. + * + * 2014 Petteri Aimonen + */ + +#include "pb_common.h" + +bool pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct) +{ + iter->start = fields; + iter->pos = fields; + iter->required_field_index = 0; + iter->dest_struct = dest_struct; + iter->pData = (char *)dest_struct + iter->pos->data_offset; + iter->pSize = (char *)iter->pData + iter->pos->size_offset; + + return (iter->pos->tag != 0); +} + +bool pb_field_iter_next(pb_field_iter_t *iter) +{ + const pb_field_t *prev_field = iter->pos; + + if (prev_field->tag == 0) + /* Handle empty message types, where the first field is already the terminator. + * In other cases, the iter->pos never points to the terminator. + */ + return false; + + iter->pos++; + + if (iter->pos->tag == 0) { + /* Wrapped back to beginning, reinitialize */ + (void)pb_field_iter_begin(iter, iter->start, iter->dest_struct); + return false; + } else { + /* Increment the pointers based on previous field size */ + size_t prev_size = prev_field->data_size; + + if (PB_HTYPE(prev_field->type) == PB_HTYPE_ONEOF && + PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF && + iter->pos->data_offset == PB_SIZE_MAX) + /* Don't advance pointers inside unions */ + return true; + else if (PB_ATYPE(prev_field->type) == PB_ATYPE_STATIC && + PB_HTYPE(prev_field->type) == PB_HTYPE_REPEATED) + /* In static arrays, the data_size tells the size of a single entry and + * array_size is the number of entries + */ + prev_size *= prev_field->array_size; + else if (PB_ATYPE(prev_field->type) == PB_ATYPE_POINTER) + /* Pointer fields always have a constant size in the main structure. + * The data_size only applies to the dynamically allocated area. + */ + prev_size = sizeof(void *); + + if (PB_HTYPE(prev_field->type) == PB_HTYPE_REQUIRED) + /* Count the required fields, in order to check their presence in the + * decoder. + */ + iter->required_field_index++; + + iter->pData = (char *)iter->pData + prev_size + iter->pos->data_offset; + iter->pSize = (char *)iter->pData + iter->pos->size_offset; + + return true; + } +} + +bool pb_field_iter_find(pb_field_iter_t *iter, uint32_t tag) +{ + const pb_field_t *start = iter->pos; + + do { + if (iter->pos->tag == tag && + PB_LTYPE(iter->pos->type) != PB_LTYPE_EXTENSION) + /* Found the wanted field */ + return true; + + (void)pb_field_iter_next(iter); + } while (iter->pos != start); + + /* Searched all the way back to start, and found nothing. */ + return false; +} + diff --git a/drivers/misc/vehicle/nanopb-c/pb_common.h b/drivers/misc/vehicle/nanopb-c/pb_common.h new file mode 100644 index 000000000000..9f0edf002f92 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_common.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_common.h: Common support functions for pb_encode.c and pb_decode.c. + * These functions are rarely needed by applications directly. + */ + +#ifndef PB_COMMON_H_INCLUDED +#define PB_COMMON_H_INCLUDED + +#include "pb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Iterator for pb_field_t list */ +struct pb_field_iter_s { + const pb_field_t *start; /* Start of the pb_field_t array */ + const pb_field_t *pos; /* Current position of the iterator */ + /* Zero-based index that counts only the required fields */ + unsigned int required_field_index; + void *dest_struct; /* Pointer to start of the structure */ + void *pData; /* Pointer to current field value */ + void *pSize; /* Pointer to count/has field */ +}; +typedef struct pb_field_iter_s pb_field_iter_t; + +/* Initialize the field iterator structure to beginning. + * Returns false if the message type is empty. + */ +bool pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct); + +/* Advance the iterator to the next field. + * Returns false when the iterator wraps back to the first field. + */ +bool pb_field_iter_next(pb_field_iter_t *iter); + +/* Advance the iterator until it points at a field with the given tag. + * Returns false if no such field exists. + */ +bool pb_field_iter_find(pb_field_iter_t *iter, uint32_t tag); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif + diff --git a/drivers/misc/vehicle/nanopb-c/pb_decode.c b/drivers/misc/vehicle/nanopb-c/pb_decode.c new file mode 100644 index 000000000000..f2b6334b1f14 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_decode.c @@ -0,0 +1,1503 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_decode.c -- decode a protobuf using minimal resources + * + * 2011 Petteri Aimonen + */ + +/* Use the GCC warn_unused_result attribute to check that all return values + * are propagated correctly. On other compilers and gcc before 3.4.0 just + * ignore the annotation. + */ +#if (!defined(__GNUC__)) || (__GNUC__ < 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ < 4)) + #define checkreturn +#else + #define checkreturn __attribute__((warn_unused_result)) +#endif + +#include "pb.h" +#include "pb_decode.h" +#include "pb_common.h" + +/************************************** + * Declarations internal to this file * + **************************************/ + +typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; + +static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); +static bool checkreturn read_raw_value(pb_istream_t *stream, + pb_wire_type_t wire_type, + pb_byte_t *buf, + size_t *size); +static bool checkreturn decode_static_field(pb_istream_t *stream, + pb_wire_type_t wire_type, + pb_field_iter_t *iter); +static bool checkreturn decode_callback_field(pb_istream_t *stream, + pb_wire_type_t wire_type, + pb_field_iter_t *iter); +static bool checkreturn decode_field(pb_istream_t *stream, + pb_wire_type_t wire_type, + pb_field_iter_t *iter); +static void iter_from_extension(pb_field_iter_t *iter, + pb_extension_t *extension); +static bool checkreturn default_extension_decoder(pb_istream_t *stream, + pb_extension_t *extension, + uint32_t tag, + pb_wire_type_t wire_type); +static bool checkreturn decode_extension(pb_istream_t *stream, + uint32_t tag, + pb_wire_type_t wire_type, + pb_field_iter_t *iter); +static bool checkreturn find_extension_field(pb_field_iter_t *iter); +static void pb_field_set_to_default(pb_field_iter_t *iter); +static void pb_message_set_to_defaults(const pb_field_t fields[], + void *dest_struct); +static bool checkreturn pb_dec_varint(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_decode_varint32_eof(pb_istream_t *stream, + uint32_t *dest, + bool *eof); +static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_svarint(pb_istream_t *stream, + const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_bytes(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_string(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_submessage(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, + const pb_field_t *field, + void *dest); +static bool checkreturn pb_skip_varint(pb_istream_t *stream); +static bool checkreturn pb_skip_string(pb_istream_t *stream); + +#ifdef PB_ENABLE_MALLOC +static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, + size_t data_size, size_t array_size); +static bool checkreturn pb_release_union_field(pb_istream_t *stream, + pb_field_iter_t *iter); +static void pb_release_single_field(const pb_field_iter_t *iter); +#endif + +#ifdef PB_WITHOUT_64BIT +#define pb_int64_t int32_t +#define pb_uint64_t uint32_t +#else +#define pb_int64_t int64_t +#define pb_uint64_t uint64_t +#endif + +/* --- Function pointers to field decoders --- + * Order in the array must match pb_action_t LTYPE numbering. + */ +static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { + &pb_dec_varint, + &pb_dec_uvarint, + &pb_dec_svarint, + &pb_dec_fixed32, + &pb_dec_fixed64, + + &pb_dec_bytes, + &pb_dec_string, + &pb_dec_submessage, + NULL, /* extensions */ + &pb_dec_fixed_length_bytes +}; + +/******************************* + * pb_istream_t implementation * + *******************************/ + +static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count) +{ + size_t i; + + const pb_byte_t *source = (const pb_byte_t *)stream->state; + + stream->state = (pb_byte_t *)stream->state + count; + if (buf != NULL) { + for (i = 0; i < count; i++) + buf[i] = source[i]; + } + + return true; +} + +bool checkreturn pb_read(pb_istream_t *stream, pb_byte_t *buf, size_t count) +{ +#ifndef PB_BUFFER_ONLY + if (buf == NULL && stream->callback != buf_read) { + /* Skip input bytes */ + pb_byte_t tmp[16]; + + while (count > 16) { + if (!pb_read(stream, tmp, 16)) + return false; + + count -= 16; + } + + return pb_read(stream, tmp, count); + } +#endif + + if (stream->bytes_left < count) + PB_RETURN_ERROR(stream, "end-of-stream"); + +#ifndef PB_BUFFER_ONLY + if (!stream->callback(stream, buf, count)) + PB_RETURN_ERROR(stream, "io error"); +#else + if (!buf_read(stream, buf, count)) + return false; +#endif + + stream->bytes_left -= count; + return true; +} + +/* Read a single byte from input stream. buf may not be NULL. + * This is an optimization for the varint decoding. + */ +static bool checkreturn pb_readbyte(pb_istream_t *stream, pb_byte_t *buf) +{ + if (stream->bytes_left == 0) + PB_RETURN_ERROR(stream, "end-of-stream"); + +#ifndef PB_BUFFER_ONLY + if (!stream->callback(stream, buf, 1)) + PB_RETURN_ERROR(stream, "io error"); +#else + *buf = *(const pb_byte_t *)stream->state; + stream->state = (pb_byte_t *)stream->state + 1; +#endif + + stream->bytes_left--; + + return true; +} + +pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize) +{ + pb_istream_t stream; + /* Cast away the const from buf without a compiler error. We are + * careful to use it only in a const manner in the callbacks. + */ + union { + void *state; + const void *c_state; + } state; + +#ifdef PB_BUFFER_ONLY + stream.callback = NULL; +#else + stream.callback = &buf_read; +#endif + state.c_state = buf; + stream.state = state.state; + stream.bytes_left = bufsize; +#ifndef PB_NO_ERRMSG + stream.errmsg = NULL; +#endif + return stream; +} + +/******************** + * Helper functions * + ********************/ + +static bool checkreturn pb_decode_varint32_eof(pb_istream_t *stream, uint32_t *dest, bool *eof) +{ + pb_byte_t byte; + uint32_t result; + + if (!pb_readbyte(stream, &byte)) { + if (stream->bytes_left == 0) { + if (eof) + *eof = true; + } + + return false; + } + + if ((byte & 0x80) == 0) + /* Quick case, 1 byte value */ + result = byte; + else { + /* Multibyte case */ + uint_fast8_t bitpos = 7; + + result = byte & 0x7F; + + do { + if (!pb_readbyte(stream, &byte)) + return false; + + if (bitpos >= 32) { + /* The varint could have trailing 0x80 bytes, + * or 0xFF for negative. + */ + uint8_t sign_extension = (bitpos < 63) ? 0xFF : 0x01; + + if ((byte & 0x7F) != 0x00 && + ((result >> 31) == 0 || byte != sign_extension)) + PB_RETURN_ERROR(stream, "varint overflow"); + } else + result |= (uint32_t)(byte & 0x7F) << bitpos; + + bitpos = (uint_fast8_t)(bitpos + 7); + } while (byte & 0x80); + + if (bitpos == 35 && (byte & 0x70) != 0) + /* The last byte was at bitpos=28, so only bottom 4 bits fit. */ + PB_RETURN_ERROR(stream, "varint overflow"); + } + + *dest = result; + return true; +} + +bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) +{ + return pb_decode_varint32_eof(stream, dest, NULL); +} + +#ifndef PB_WITHOUT_64BIT +bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) +{ + pb_byte_t byte; + uint_fast8_t bitpos = 0; + uint64_t result = 0; + + do { + if (bitpos >= 64) + PB_RETURN_ERROR(stream, "varint overflow"); + + if (!pb_readbyte(stream, &byte)) + return false; + + result |= (uint64_t)(byte & 0x7F) << bitpos; + bitpos = (uint_fast8_t)(bitpos + 7); + } while (byte & 0x80); + + *dest = result; + return true; +} +#endif + +bool checkreturn pb_skip_varint(pb_istream_t *stream) +{ + pb_byte_t byte; + + do { + if (!pb_read(stream, &byte, 1)) + return false; + } while (byte & 0x80); + return true; +} + +bool checkreturn pb_skip_string(pb_istream_t *stream) +{ + uint32_t length; + + if (!pb_decode_varint32(stream, &length)) + return false; + + return pb_read(stream, NULL, length); +} + +bool checkreturn pb_decode_tag(pb_istream_t *stream, + pb_wire_type_t *wire_type, uint32_t *tag, bool *eof) +{ + uint32_t temp; + *eof = false; + *wire_type = (pb_wire_type_t) 0; + *tag = 0; + + if (!pb_decode_varint32_eof(stream, &temp, eof)) + return false; + + if (temp == 0) { + *eof = true; /* Special feature: allow 0-terminated messages. */ + return false; + } + + *tag = temp >> 3; + *wire_type = (pb_wire_type_t)(temp & 7); + + return true; +} + +bool checkreturn pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type) +{ + switch (wire_type) { + case PB_WT_VARINT: return pb_skip_varint(stream); + case PB_WT_64BIT: return pb_read(stream, NULL, 8); + case PB_WT_STRING: return pb_skip_string(stream); + case PB_WT_32BIT: return pb_read(stream, NULL, 4); + default: + PB_RETURN_ERROR(stream, "invalid wire_type"); + } +} + +/* Read a raw value to buffer, for the purpose of passing it to callback as + * a substream. Size is maximum size on call, and actual size on return. + */ +static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, + pb_byte_t *buf, size_t *size) +{ + size_t max_size = *size; + + switch (wire_type) { + case PB_WT_VARINT: + *size = 0; + do { + (*size)++; + if (*size > max_size) + return false; + if (!pb_read(stream, buf, 1)) + return false; + } while (*buf++ & 0x80); + return true; + + case PB_WT_64BIT: + *size = 8; + return pb_read(stream, buf, 8); + + case PB_WT_32BIT: + *size = 4; + return pb_read(stream, buf, 4); + + case PB_WT_STRING: + /* Calling read_raw_value with a PB_WT_STRING is an error. + * Explicitly handle this case and fallthrough to default to avoid + * compiler warnings. + */ + + default: + PB_RETURN_ERROR(stream, "invalid wire_type"); + } +} + +/* Decode string length from stream and return a substream with limited length. + * Remember to close the substream using pb_close_string_substream(). + */ +bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream) +{ + uint32_t size; + + if (!pb_decode_varint32(stream, &size)) + return false; + + *substream = *stream; + if (substream->bytes_left < size) + PB_RETURN_ERROR(stream, "parent stream too short"); + + substream->bytes_left = size; + stream->bytes_left -= size; + return true; +} + +bool checkreturn pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) +{ + if (substream->bytes_left) { + if (!pb_read(substream, NULL, substream->bytes_left)) + return false; + } + + stream->state = substream->state; + +#ifndef PB_NO_ERRMSG + stream->errmsg = substream->errmsg; +#endif + return true; +} + +/************************* + * Decode a single field * + *************************/ + +static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, + pb_field_iter_t *iter) +{ + pb_type_t type; + pb_decoder_t func; + + type = iter->pos->type; + func = PB_DECODERS[PB_LTYPE(type)]; + + switch (PB_HTYPE(type)) { + case PB_HTYPE_REQUIRED: + return func(stream, iter->pos, iter->pData); + + case PB_HTYPE_OPTIONAL: + if (iter->pSize != iter->pData) + *(bool *)iter->pSize = true; + return func(stream, iter->pos, iter->pData); + + case PB_HTYPE_REPEATED: + if (wire_type == PB_WT_STRING + && PB_LTYPE(type) <= PB_LTYPE_LAST_PACKABLE) { + /* Packed array */ + bool status = true; + pb_size_t *size = (pb_size_t *)iter->pSize; + pb_istream_t substream; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + while (substream.bytes_left > 0 && *size < iter->pos->array_size) { + void *pItem = (char *)iter->pData + iter->pos->data_size * (*size); + + if (!func(&substream, iter->pos, pItem)) { + status = false; + break; + } + (*size)++; + } + + if (substream.bytes_left != 0) + PB_RETURN_ERROR(stream, "array overflow"); + if (!pb_close_string_substream(stream, &substream)) + return false; + + return status; + } else { + /* Repeated field */ + pb_size_t *size = (pb_size_t *)iter->pSize; + char *pItem = (char *)iter->pData + iter->pos->data_size * (*size); + + if ((*size)++ >= iter->pos->array_size) + PB_RETURN_ERROR(stream, "array overflow"); + + return func(stream, iter->pos, pItem); + } + + case PB_HTYPE_ONEOF: + *(pb_size_t *)iter->pSize = iter->pos->tag; + if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) { + /* We memset to zero so that any callbacks are set to NULL. + * Then set any default values. + */ + memset(iter->pData, 0, iter->pos->data_size); + pb_message_set_to_defaults((const pb_field_t *)iter->pos->ptr, iter->pData); + } + return func(stream, iter->pos, iter->pData); + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +} + +#ifdef PB_ENABLE_MALLOC +/* Allocate storage for the field and store the pointer at iter->pData. + * array_size is the number of entries to reserve in an array. + * Zero size is not allowed, use pb_free() for releasing. + */ +static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, + size_t array_size) +{ + void *ptr = *(void **)pData; + + if (data_size == 0 || array_size == 0) + PB_RETURN_ERROR(stream, "invalid size"); + + /* Check for multiplication overflows. + * This code avoids the costly division if the sizes are small enough. + * Multiplication is safe as long as only half of bits are set + * in either multiplicand. + */ + const size_t check_limit = (size_t)1 << (sizeof(size_t) * 4); + + if (data_size >= check_limit || array_size >= check_limit) { + const size_t size_max = (size_t)-1; + + if (size_max / array_size < data_size) + PB_RETURN_ERROR(stream, "size too large"); + } + + /* Allocate new or expand previous allocation */ + /* Note: on failure the old pointer will remain in the structure, + * the message must be freed by caller also on error return. + */ + ptr = pb_realloc(ptr, array_size * data_size); + if (ptr == NULL) + PB_RETURN_ERROR(stream, "realloc failed"); + + *(void **)pData = ptr; + return true; +} + +/* Clear a newly allocated item in case it contains a pointer, or is a submessage. */ +static void initialize_pointer_field(void *pItem, pb_field_iter_t *iter) +{ + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_STRING || + PB_LTYPE(iter->pos->type) == PB_LTYPE_BYTES) + *(void **)pItem = NULL; + else if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE) { + /* We memset to zero so that any callbacks are set to NULL. + * Then set any default values. + */ + memset(pItem, 0, iter->pos->data_size); + pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, pItem); + } +} +#endif + +static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, + pb_field_iter_t *iter) +{ +#ifndef PB_ENABLE_MALLOC + PB_UNUSED(wire_type); + PB_UNUSED(iter); + PB_RETURN_ERROR(stream, "no malloc support"); +#else + pb_type_t type; + pb_decoder_t func; + + type = iter->pos->type; + func = PB_DECODERS[PB_LTYPE(type)]; + + switch (PB_HTYPE(type)) { + case PB_HTYPE_REQUIRED: + case PB_HTYPE_OPTIONAL: + case PB_HTYPE_ONEOF: + if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE && + *(void **)iter->pData != NULL) + /* Duplicate field, have to release the old allocation first. */ + pb_release_single_field(iter); + + if (PB_HTYPE(type) == PB_HTYPE_ONEOF) + *(pb_size_t *)iter->pSize = iter->pos->tag; + + if (PB_LTYPE(type) == PB_LTYPE_STRING || + PB_LTYPE(type) == PB_LTYPE_BYTES) + return func(stream, iter->pos, iter->pData); + else { + if (!allocate_field(stream, iter->pData, iter->pos->data_size, 1)) + return false; + + initialize_pointer_field(*(void **)iter->pData, iter); + return func(stream, iter->pos, *(void **)iter->pData); + } + + case PB_HTYPE_REPEATED: + if (wire_type == PB_WT_STRING + && PB_LTYPE(type) <= PB_LTYPE_LAST_PACKABLE) { + /* Packed array, multiple items come in at once. */ + bool status = true; + pb_size_t *size = (pb_size_t *)iter->pSize; + size_t allocated_size = *size; + void *pItem; + pb_istream_t substream; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + while (substream.bytes_left) { + if ((size_t)*size + 1 > allocated_size) { + /* Allocate more storage. This tries to guess the + * number of remaining entries. Round the division + * upwards. + */ + allocated_size += (substream.bytes_left - 1) / + iter->pos->data_size + 1; + + if (!allocate_field(&substream, iter->pData, + iter->pos->data_size, allocated_size)) { + status = false; + break; + } + } + + /* Decode the array entry */ + pItem = *(char **)iter->pData + iter->pos->data_size * (*size); + initialize_pointer_field(pItem, iter); + if (!func(&substream, iter->pos, pItem)) { + status = false; + break; + } + + if (*size == PB_SIZE_MAX) { +#ifndef PB_NO_ERRMSG + stream->errmsg = "too many array entries"; +#endif + status = false; + break; + } + + (*size)++; + } + if (!pb_close_string_substream(stream, &substream)) + return false; + + return status; + } else { + /* Normal repeated field, i.e. only one item at a time. */ + pb_size_t *size = (pb_size_t *)iter->pSize; + void *pItem; + + if (*size == PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "too many array entries"); + + (*size)++; + if (!allocate_field(stream, iter->pData, iter->pos->data_size, *size)) + return false; + + pItem = *(char **)iter->pData + iter->pos->data_size * (*size - 1); + initialize_pointer_field(pItem, iter); + return func(stream, iter->pos, pItem); + } + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +#endif +} + +static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, + pb_field_iter_t *iter) +{ + pb_callback_t *pCallback = (pb_callback_t *)iter->pData; + +#ifdef PB_OLD_CALLBACK_STYLE + void *arg = pCallback->arg; +#else + void **arg = &(pCallback->arg); +#endif + + if (pCallback == NULL || pCallback->funcs.decode == NULL) + return pb_skip_field(stream, wire_type); + + if (wire_type == PB_WT_STRING) { + pb_istream_t substream; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + do { + if (!pCallback->funcs.decode(&substream, iter->pos, arg)) + PB_RETURN_ERROR(stream, "callback failed"); + } while (substream.bytes_left); + + if (!pb_close_string_substream(stream, &substream)) + return false; + + return true; + } else { + /* Copy the single scalar value to stack. + * This is required so that we can limit the stream length, + * which in turn allows to use same callback for packed and + * not-packed fields. + */ + pb_istream_t substream; + pb_byte_t buffer[10]; + size_t size = sizeof(buffer); + + if (!read_raw_value(stream, wire_type, buffer, &size)) + return false; + substream = pb_istream_from_buffer(buffer, size); + + return pCallback->funcs.decode(&substream, iter->pos, arg); + } +} + +static bool checkreturn decode_field(pb_istream_t *stream, + pb_wire_type_t wire_type, pb_field_iter_t *iter) +{ +#ifdef PB_ENABLE_MALLOC + /* When decoding an oneof field, check if there is old data that must be + * released first. + */ + if (PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF) { + if (!pb_release_union_field(stream, iter)) + return false; + } +#endif + + switch (PB_ATYPE(iter->pos->type)) { + case PB_ATYPE_STATIC: + return decode_static_field(stream, wire_type, iter); + + case PB_ATYPE_POINTER: + return decode_pointer_field(stream, wire_type, iter); + + case PB_ATYPE_CALLBACK: + return decode_callback_field(stream, wire_type, iter); + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +} + +static void iter_from_extension(pb_field_iter_t *iter, pb_extension_t *extension) +{ + /* Fake a field iterator for the extension field. + * It is not actually safe to advance this iterator, but decode_field + * will not even try to. + */ + const pb_field_t *field = (const pb_field_t *)extension->type->arg; + (void)pb_field_iter_begin(iter, field, extension->dest); + iter->pData = extension->dest; + iter->pSize = &extension->found; + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + /* For pointer extensions, the pointer is stored directly + * in the extension structure. This avoids having an extra + * indirection. + */ + iter->pData = &extension->dest; +} + +/* Default handler for extension fields. Expects a pb_field_t structure + * in extension->type->arg. + */ +static bool checkreturn default_extension_decoder(pb_istream_t *stream, + pb_extension_t *extension, + uint32_t tag, pb_wire_type_t wire_type) +{ + const pb_field_t *field = (const pb_field_t *)extension->type->arg; + pb_field_iter_t iter; + + if (field->tag != tag) + return true; + + iter_from_extension(&iter, extension); + extension->found = true; + return decode_field(stream, wire_type, &iter); +} + +/* Try to decode an unknown field as an extension field. Tries each extension + * decoder in turn, until one of them handles the field or loop ends. + */ +static bool checkreturn decode_extension(pb_istream_t *stream, + uint32_t tag, pb_wire_type_t wire_type, + pb_field_iter_t *iter) +{ + pb_extension_t *extension = *(pb_extension_t * const *)iter->pData; + size_t pos = stream->bytes_left; + + while (extension != NULL && pos == stream->bytes_left) { + bool status; + + if (extension->type->decode) + status = extension->type->decode(stream, extension, tag, wire_type); + else + status = default_extension_decoder(stream, extension, tag, wire_type); + + if (!status) + return false; + + extension = extension->next; + } + + return true; +} + +/* Step through the iterator until an extension field is found or until all + * entries have been checked. There can be only one extension field per + * message. Returns false if no extension field is found. + */ +static bool checkreturn find_extension_field(pb_field_iter_t *iter) +{ + const pb_field_t *start = iter->pos; + + do { + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_EXTENSION) + return true; + (void)pb_field_iter_next(iter); + } while (iter->pos != start); + + return false; +} + +/* Initialize message fields to default values, recursively */ +static void pb_field_set_to_default(pb_field_iter_t *iter) +{ + pb_type_t type; + + type = iter->pos->type; + if (PB_LTYPE(type) == PB_LTYPE_EXTENSION) { + pb_extension_t *ext = *(pb_extension_t * const *)iter->pData; + + while (ext != NULL) { + pb_field_iter_t ext_iter; + + ext->found = false; + iter_from_extension(&ext_iter, ext); + pb_field_set_to_default(&ext_iter); + ext = ext->next; + } + } else if (PB_ATYPE(type) == PB_ATYPE_STATIC) { + bool init_data = true; + + if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL && iter->pSize != iter->pData) + /* Set has_field to false. Still initialize the optional field + * itself also. + */ + *(bool *)iter->pSize = false; + else if (PB_HTYPE(type) == PB_HTYPE_REPEATED || + PB_HTYPE(type) == PB_HTYPE_ONEOF) { + /* REPEATED: Set array count to 0, no need to initialize contents. + *ONEOF: Set which_field to 0. + */ + *(pb_size_t *)iter->pSize = 0; + init_data = false; + } + + if (init_data) { + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE) + /* Initialize submessage to defaults */ + pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, + iter->pData); + else if (iter->pos->ptr != NULL) + /* Initialize to default value */ + memcpy(iter->pData, iter->pos->ptr, iter->pos->data_size); + else + /* Initialize to zeros */ + memset(iter->pData, 0, iter->pos->data_size); + } + } else if (PB_ATYPE(type) == PB_ATYPE_POINTER) { + /* Initialize the pointer to NULL. */ + *(void **)iter->pData = NULL; + + /* Initialize array count to 0. */ + if (PB_HTYPE(type) == PB_HTYPE_REPEATED || + PB_HTYPE(type) == PB_HTYPE_ONEOF) + *(pb_size_t *)iter->pSize = 0; + } +} + +static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) +{ + pb_field_iter_t iter; + + if (!pb_field_iter_begin(&iter, fields, dest_struct)) + return; /* Empty message type */ + + do { + pb_field_set_to_default(&iter); + } while (pb_field_iter_next(&iter)); +} + +/********************* + * Decode all fields * + *********************/ + +bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], + void *dest_struct) +{ + uint32_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 31) / 32] = {0, 0}; + const uint32_t allbits = ~(uint32_t)0; + uint32_t extension_range_start = 0; + pb_field_iter_t iter; + + /* 'fixed_count_field' and 'fixed_count_size' track position of a repeated fixed + * count field. This can only handle _one_ repeated fixed count field that + * is unpacked and unordered among other (non repeated fixed count) fields. + */ + const pb_field_t *fixed_count_field = NULL; + pb_size_t fixed_count_size = 0; + + /* Return value ignored, as empty message types will be correctly handled by + * pb_field_iter_find() anyway. + */ + (void)pb_field_iter_begin(&iter, fields, dest_struct); + + while (stream->bytes_left) { + uint32_t tag; + pb_wire_type_t wire_type; + bool eof; + + if (!pb_decode_tag(stream, &wire_type, &tag, &eof)) { + if (eof) + break; + else + return false; + } + + if (!pb_field_iter_find(&iter, tag)) { + /* No match found, check if it matches an extension. */ + if (tag >= extension_range_start) { + if (!find_extension_field(&iter)) + extension_range_start = (uint32_t)-1; + else + extension_range_start = iter.pos->tag; + + if (tag >= extension_range_start) { + size_t pos = stream->bytes_left; + + if (!decode_extension(stream, tag, wire_type, &iter)) + return false; + + if (pos != stream->bytes_left) + /* The field was handled */ + continue; + } + } + + /* No match found, skip data */ + if (!pb_skip_field(stream, wire_type)) + return false; + continue; + } + + /* If a repeated fixed count field was found, get size from + * 'fixed_count_field' as there is no counter contained in the struct. + */ + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REPEATED + && iter.pSize == iter.pData) { + if (fixed_count_field != iter.pos) { + /* If the new fixed count field does not match the previous one, + * check that the previous one is NULL or that it finished + * receiving all the expected data. + */ + if (fixed_count_field != NULL && + fixed_count_size != fixed_count_field->array_size) + PB_RETURN_ERROR(stream, "wrong size for fixed count field"); + + fixed_count_field = iter.pos; + fixed_count_size = 0; + } + + iter.pSize = &fixed_count_size; + } + + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REQUIRED + && iter.required_field_index < PB_MAX_REQUIRED_FIELDS) { + uint32_t tmp = ((uint32_t)1 << (iter.required_field_index & 31)); + + fields_seen[iter.required_field_index >> 5] |= tmp; + } + + if (!decode_field(stream, wire_type, &iter)) + return false; + } + + /* Check that all elements of the last decoded fixed count field were present. */ + if (fixed_count_field != NULL && + fixed_count_size != fixed_count_field->array_size) + PB_RETURN_ERROR(stream, "wrong size for fixed count field"); + + /* Check that all required fields were present. */ + /* First figure out the number of required fields by + * seeking to the end of the field array. Usually we + * are already close to end after decoding. + */ + unsigned int req_field_count; + pb_type_t last_type; + unsigned int i; + + do { + req_field_count = iter.required_field_index; + last_type = iter.pos->type; + } while (pb_field_iter_next(&iter)); + + /* Fixup if last field was also required. */ + if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag != 0) + req_field_count++; + + if (req_field_count > PB_MAX_REQUIRED_FIELDS) + req_field_count = PB_MAX_REQUIRED_FIELDS; + + if (req_field_count > 0) { + /* Check the whole words */ + for (i = 0; i < (req_field_count >> 5); i++) { + if (fields_seen[i] != allbits) + PB_RETURN_ERROR(stream, "missing required field"); + } + + /* Check the remaining bits (if any) */ + if ((req_field_count & 31) != 0) { + if (fields_seen[req_field_count >> 5] != + (allbits >> (32 - (req_field_count & 31)))) + PB_RETURN_ERROR(stream, "missing required field"); + } + } + + return true; +} + +bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + bool status; + + pb_message_set_to_defaults(fields, dest_struct); + status = pb_decode_noinit(stream, fields, dest_struct); + +#ifdef PB_ENABLE_MALLOC + if (!status) + pb_release(fields, dest_struct); +#endif + + return status; +} + +bool pb_decode_delimited_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + pb_istream_t substream; + bool status; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + status = pb_decode_noinit(&substream, fields, dest_struct); + + if (!pb_close_string_substream(stream, &substream)) + return false; + return status; +} + +bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + pb_istream_t substream; + bool status; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + status = pb_decode(&substream, fields, dest_struct); + + if (!pb_close_string_substream(stream, &substream)) + return false; + return status; +} + +bool pb_decode_nullterminated(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + /* This behaviour will be separated in nanopb-0.4.0, see issue #278. */ + return pb_decode(stream, fields, dest_struct); +} + +#ifdef PB_ENABLE_MALLOC +/* Given an oneof field, if there has already been a field inside this oneof, + * release it before overwriting with a different one. + */ +static bool pb_release_union_field(pb_istream_t *stream, pb_field_iter_t *iter) +{ + pb_size_t old_tag = *(pb_size_t *)iter->pSize; /* Previous which_ value */ + pb_size_t new_tag = iter->pos->tag; /* New which_ value */ + + if (old_tag == 0) + return true; /* Ok, no old data in union */ + + if (old_tag == new_tag) + return true; /* Ok, old data is of same type => merge */ + + /* Release old data. The find can fail if the message struct contains + * invalid data. + */ + if (!pb_field_iter_find(iter, old_tag)) + PB_RETURN_ERROR(stream, "invalid union tag"); + + pb_release_single_field(iter); + + /* Restore iterator to where it should be. + * This shouldn't fail unless the pb_field_t structure is corrupted. + */ + if (!pb_field_iter_find(iter, new_tag)) + PB_RETURN_ERROR(stream, "iterator error"); + + return true; +} + +static void pb_release_single_field(const pb_field_iter_t *iter) +{ + pb_type_t type; + + type = iter->pos->type; + if (PB_HTYPE(type) == PB_HTYPE_ONEOF) { + if (*(pb_size_t *)iter->pSize != iter->pos->tag) + return; /* This is not the current field in the union */ + } + + /* Release anything contained inside an extension or submsg. + * This has to be done even if the submsg itself is statically + * allocated. + */ + if (PB_LTYPE(type) == PB_LTYPE_EXTENSION) { + /* Release fields from all extensions in the linked list */ + pb_extension_t *ext = *(pb_extension_t **)iter->pData; + + while (ext != NULL) { + pb_field_iter_t ext_iter; + + iter_from_extension(&ext_iter, ext); + pb_release_single_field(&ext_iter); + ext = ext->next; + } + } else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) { + /* Release fields in submessage or submsg array */ + void *pItem = iter->pData; + pb_size_t count = 1; + + if (PB_ATYPE(type) == PB_ATYPE_POINTER) + pItem = *(void **)iter->pData; + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) { + if (PB_ATYPE(type) == PB_ATYPE_STATIC && iter->pSize == iter->pData) + /* No _count field so use size of the array */ + count = iter->pos->array_size; + else + count = *(pb_size_t *)iter->pSize; + + if (PB_ATYPE(type) == PB_ATYPE_STATIC && count > iter->pos->array_size) + /* Protect against corrupted _count fields */ + count = iter->pos->array_size; + } + + if (pItem) { + while (count--) { + pb_release((const pb_field_t *)iter->pos->ptr, pItem); + pItem = (char *)pItem + iter->pos->data_size; + } + } + } + + if (PB_ATYPE(type) == PB_ATYPE_POINTER) { + if (PB_HTYPE(type) == PB_HTYPE_REPEATED && + (PB_LTYPE(type) == PB_LTYPE_STRING || + PB_LTYPE(type) == PB_LTYPE_BYTES)) { + /* Release entries in repeated string or bytes array */ + void **pItem = *(void ***)iter->pData; + pb_size_t count = *(pb_size_t *)iter->pSize; + + while (count--) { + pb_free(*pItem); + *pItem++ = NULL; + } + } + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + /* We are going to release the array, so set the size to 0 */ + *(pb_size_t *)iter->pSize = 0; + + /* Release main item */ + pb_free(*(void **)iter->pData); + *(void **)iter->pData = NULL; + } +} + +void pb_release(const pb_field_t fields[], void *dest_struct) +{ + pb_field_iter_t iter; + + if (!dest_struct) + return; /* Ignore NULL pointers, similar to free() */ + + if (!pb_field_iter_begin(&iter, fields, dest_struct)) + return; /* Empty message type */ + + do { + pb_release_single_field(&iter); + } while (pb_field_iter_next(&iter)); +} +#endif + +/* Field decoders */ + +bool pb_decode_svarint(pb_istream_t *stream, pb_int64_t *dest) +{ + pb_uint64_t value; + + if (!pb_decode_varint(stream, &value)) + return false; + + if (value & 1) + *dest = (pb_int64_t)(~(value >> 1)); + else + *dest = (pb_int64_t)(value >> 1); + + return true; +} + +bool pb_decode_fixed32(pb_istream_t *stream, void *dest) +{ + pb_byte_t bytes[4]; + + if (!pb_read(stream, bytes, 4)) + return false; + + *(uint32_t *)dest = ((uint32_t)bytes[0] << 0) | + ((uint32_t)bytes[1] << 8) | + ((uint32_t)bytes[2] << 16) | + ((uint32_t)bytes[3] << 24); + return true; +} + +#ifndef PB_WITHOUT_64BIT +bool pb_decode_fixed64(pb_istream_t *stream, void *dest) +{ + pb_byte_t bytes[8]; + + if (!pb_read(stream, bytes, 8)) + return false; + + *(uint64_t *)dest = ((uint64_t)bytes[0] << 0) | + ((uint64_t)bytes[1] << 8) | + ((uint64_t)bytes[2] << 16) | + ((uint64_t)bytes[3] << 24) | + ((uint64_t)bytes[4] << 32) | + ((uint64_t)bytes[5] << 40) | + ((uint64_t)bytes[6] << 48) | + ((uint64_t)bytes[7] << 56); + + return true; +} +#endif + +static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + pb_uint64_t value; + pb_int64_t svalue; + pb_int64_t clamped; + + if (!pb_decode_varint(stream, &value)) + return false; + + /* See issue 97: Google's C++ protobuf allows negative varint values to + * be cast as int32_t, instead of the int64_t that should be used when + * encoding. Previous nanopb versions had a bug in encoding. In order to + * not break decoding of such messages, we cast <=32 bit fields to + * int32_t first to get the sign correct. + */ + if (field->data_size == sizeof(pb_int64_t)) + svalue = (pb_int64_t)value; + else + svalue = (int32_t)value; + + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(pb_int64_t)) + clamped = *(pb_int64_t *)dest = svalue; + else if (field->data_size == sizeof(int32_t)) + clamped = *(int32_t *)dest = (int32_t)svalue; + else if (field->data_size == sizeof(int_least16_t)) + clamped = *(int_least16_t *)dest = (int_least16_t)svalue; + else if (field->data_size == sizeof(int_least8_t)) + clamped = *(int_least8_t *)dest = (int_least8_t)svalue; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + if (clamped != svalue) + PB_RETURN_ERROR(stream, "integer too large"); + + return true; +} + +static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + pb_uint64_t value, clamped; + + if (!pb_decode_varint(stream, &value)) + return false; + + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(pb_uint64_t)) + clamped = *(pb_uint64_t *)dest = value; + else if (field->data_size == sizeof(uint32_t)) + clamped = *(uint32_t *)dest = (uint32_t)value; + else if (field->data_size == sizeof(uint_least16_t)) + clamped = *(uint_least16_t *)dest = (uint_least16_t)value; + else if (field->data_size == sizeof(uint_least8_t)) + clamped = *(uint_least8_t *)dest = (uint_least8_t)value; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + if (clamped != value) + PB_RETURN_ERROR(stream, "integer too large"); + + return true; +} + +static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + pb_int64_t value, clamped; + + if (!pb_decode_svarint(stream, &value)) + return false; + + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(pb_int64_t)) + clamped = *(pb_int64_t *)dest = value; + else if (field->data_size == sizeof(int32_t)) + clamped = *(int32_t *)dest = (int32_t)value; + else if (field->data_size == sizeof(int_least16_t)) + clamped = *(int_least16_t *)dest = (int_least16_t)value; + else if (field->data_size == sizeof(int_least8_t)) + clamped = *(int_least8_t *)dest = (int_least8_t)value; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + if (clamped != value) + PB_RETURN_ERROR(stream, "integer too large"); + + return true; +} + +static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + PB_UNUSED(field); + return pb_decode_fixed32(stream, dest); +} + +static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + PB_UNUSED(field); +#ifndef PB_WITHOUT_64BIT + return pb_decode_fixed64(stream, dest); +#else + PB_UNUSED(dest); + PB_RETURN_ERROR(stream, "no 64bit support"); +#endif +} + +static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint32_t size; + size_t alloc_size; + pb_bytes_array_t *bdest; + + if (!pb_decode_varint32(stream, &size)) + return false; + + if (size > PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "bytes overflow"); + + alloc_size = PB_BYTES_ARRAY_T_ALLOCSIZE(size); + if (size > alloc_size) + PB_RETURN_ERROR(stream, "size too large"); + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { +#ifndef PB_ENABLE_MALLOC + PB_RETURN_ERROR(stream, "no malloc support"); +#else + if (!allocate_field(stream, dest, alloc_size, 1)) + return false; + bdest = *(pb_bytes_array_t **)dest; +#endif + } else { + if (alloc_size > field->data_size) + PB_RETURN_ERROR(stream, "bytes overflow"); + bdest = (pb_bytes_array_t *)dest; + } + + bdest->size = (pb_size_t)size; + return pb_read(stream, bdest->bytes, size); +} + +static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint32_t size; + size_t alloc_size; + bool status; + + if (!pb_decode_varint32(stream, &size)) + return false; + + /* Space for null terminator */ + alloc_size = size + 1; + + if (alloc_size < size) + PB_RETURN_ERROR(stream, "size too large"); + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { +#ifndef PB_ENABLE_MALLOC + PB_RETURN_ERROR(stream, "no malloc support"); +#else + if (!allocate_field(stream, dest, alloc_size, 1)) + return false; + dest = *(void **)dest; +#endif + } else { + if (alloc_size > field->data_size) + PB_RETURN_ERROR(stream, "string overflow"); + } + + status = pb_read(stream, (pb_byte_t *)dest, size); + *((pb_byte_t *)dest + size) = 0; + return status; +} + +static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + bool status; + pb_istream_t substream; + const pb_field_t *submsg_fields = (const pb_field_t *)field->ptr; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + if (field->ptr == NULL) + PB_RETURN_ERROR(stream, "invalid field descriptor"); + + /* New array entries need to be initialized, while required and optional + * submessages have already been initialized in the top-level pb_decode. + */ + if (PB_HTYPE(field->type) == PB_HTYPE_REPEATED) + status = pb_decode(&substream, submsg_fields, dest); + else + status = pb_decode_noinit(&substream, submsg_fields, dest); + + if (!pb_close_string_substream(stream, &substream)) + return false; + return status; +} + +static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_t *field, + void *dest) +{ + uint32_t size; + + if (!pb_decode_varint32(stream, &size)) + return false; + + if (size > PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "bytes overflow"); + + if (size == 0) { + /* As a special case, treat empty bytes string + * as all zeros for fixed_length_bytes. + */ + memset(dest, 0, field->data_size); + return true; + } + + if (size != field->data_size) + PB_RETURN_ERROR(stream, "incorrect fixed length bytes size"); + + return pb_read(stream, (pb_byte_t *)dest, field->data_size); +} diff --git a/drivers/misc/vehicle/nanopb-c/pb_decode.h b/drivers/misc/vehicle/nanopb-c/pb_decode.h new file mode 100644 index 000000000000..55dee5750e98 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_decode.h @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_decode.h: Functions to decode protocol buffers. Depends on pb_decode.c. + * The main function is pb_decode. You also need an input stream, and the + * field descriptions created by nanopb_generator.py. + */ + +#ifndef PB_DECODE_H_INCLUDED +#define PB_DECODE_H_INCLUDED + +#include "pb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Structure for defining custom input streams. You will need to provide + * a callback function to read the bytes from your storage, which can be + * for example a file or a network socket. + * + * The callback must conform to these rules: + * + * 1) Return false on IO errors. This will cause decoding to abort. + * 2) You can use state to store your own data (e.g. buffer pointer), + * and rely on pb_read to verify that no-body reads past bytes_left. + * 3) Your callback may be used with substreams, in which case bytes_left + * is different than from the main stream. Don't use bytes_left to compute + * any pointers. + */ +struct pb_istream_s { +#ifdef PB_BUFFER_ONLY + /* Callback pointer is not used in buffer-only configuration. + * Having an int pointer here allows binary compatibility but + * gives an error if someone tries to assign callback function. + */ + int *callback; +#else + bool (*callback)(pb_istream_t *stream, pb_byte_t *buf, size_t count); +#endif + + void *state; /* Free field for use by callback implementation */ + size_t bytes_left; + +#ifndef PB_NO_ERRMSG + const char *errmsg; +#endif +}; + +/*************************** + * Main decoding functions * + ***************************/ +/* Decode a single protocol buffers message from input stream into a C structure. + * Returns true on success, false on any failure. + * The actual struct pointed to by dest must match the description in fields. + * Callback fields of the destination structure must be initialized by caller. + * All other fields will be initialized by this function. + * + * Example usage: + * MyMessage msg = {}; + * uint8_t buffer[64]; + * pb_istream_t stream; + * + * // ... read some data into buffer ... + * + * stream = pb_istream_from_buffer(buffer, count); + * pb_decode(&stream, MyMessage_fields, &msg); + */ +bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +/* Same as pb_decode, except does not initialize the destination structure + * to default values. This is slightly faster if you need no default values + * and just do memset(struct, 0, sizeof(struct)) yourself. + * + * This can also be used for 'merging' two messages, i.e. update only the + * fields that exist in the new message. + * + * Note: If this function returns with an error, it will not release any + * dynamically allocated fields. You will need to call pb_release() yourself. + */ +bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +/* Same as pb_decode, except expects the stream to start with the message size + * encoded as varint. Corresponds to parseDelimitedFrom() in Google's + * protobuf API. + */ +bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +/* Same as pb_decode_delimited, except that it does not initialize the destination structure. + * See pb_decode_noinit + */ +bool pb_decode_delimited_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +/* Same as pb_decode, except allows the message to be terminated with a null byte. + * NOTE: Until nanopb-0.4.0, pb_decode() also allows null-termination. This behaviour + * is not supported in most other protobuf implementations, so pb_decode_delimited() + * is a better option for compatibility. + */ +bool pb_decode_nullterminated(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +#ifdef PB_ENABLE_MALLOC +/* Release any allocated pointer fields. If you use dynamic allocation, you should + * call this for any successfully decoded message when you are done with it. If + * pb_decode() returns with an error, the message is already released. + */ +void pb_release(const pb_field_t fields[], void *dest_struct); +#endif + + +/************************************** + * Functions for manipulating streams * + **************************************/ + +/* Create an input stream for reading from a memory buffer. + * + * Alternatively, you can use a custom stream that reads directly from e.g. + * a file or a network socket. + */ +pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize); + +/* Function to read from a pb_istream_t. You can use this if you need to + * read some custom header data, or to read data in field callbacks. + */ +bool pb_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); + + +/************************************************ + * Helper functions for writing field callbacks * + ************************************************/ + +/* Decode the tag for the next field in the stream. Gives the wire type and + * field tag. At end of the message, returns false and sets eof to true. + */ +bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, uint32_t *tag, bool *eof); + +/* Skip the field payload data, given the wire type. */ +bool pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type); + +/* Decode an integer in the varint format. This works for bool, enum, int32, + * int64, uint32 and uint64 field types. + */ +#ifndef PB_WITHOUT_64BIT +bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); +#else +#define pb_decode_varint pb_decode_varint32 +#endif + +/* Decode an integer in the varint format. This works for bool, enum, int32, + * and uint32 field types. + */ +bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); + +/* Decode an integer in the zig-zagged svarint format. This works for sint32 + * and sint64. + */ +#ifndef PB_WITHOUT_64BIT +bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest); +#else +bool pb_decode_svarint(pb_istream_t *stream, int32_t *dest); +#endif + +/* Decode a fixed32, sfixed32 or float value. You need to pass a pointer to + * a 4-byte wide C variable. + */ +bool pb_decode_fixed32(pb_istream_t *stream, void *dest); + +#ifndef PB_WITHOUT_64BIT +/* Decode a fixed64, sfixed64 or double value. You need to pass a pointer to + * a 8-byte wide C variable. + */ +bool pb_decode_fixed64(pb_istream_t *stream, void *dest); +#endif + +/* Make a limited-length substream for reading a PB_WT_STRING field. */ +bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream); +bool pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/drivers/misc/vehicle/nanopb-c/pb_encode.c b/drivers/misc/vehicle/nanopb-c/pb_encode.c new file mode 100644 index 000000000000..35bae5f2e08d --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_encode.c @@ -0,0 +1,889 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_encode.c -- encode a protobuf using minimal resources + * + * 2011 Petteri Aimonen + */ + +#include "pb.h" +#include "pb_encode.h" +#include "pb_common.h" + +/* Use the GCC warn_unused_result attribute to check that all return values + * are propagated correctly. On other compilers and gcc before 3.4.0 just + * ignore the annotation. + */ +#if !defined(__GNUC__) || (__GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4) + #define checkreturn +#else + #define checkreturn __attribute__((warn_unused_result)) +#endif + +/************************************** + * Declarations internal to this file * + **************************************/ +typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, + const void *src) checkreturn; + +static bool checkreturn buf_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); +static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, + const void *pData, size_t count, pb_encoder_t func); +static bool checkreturn encode_field(pb_ostream_t *stream, + const pb_field_t *field, const void *pData); +static bool checkreturn default_extension_encoder(pb_ostream_t *stream, + const pb_extension_t *extension); +static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, + const void *pData); +static void *pb_const_cast(const void *p); +static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, + const void *src); +static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, + const void *src); +static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, + const void *src); +static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, + const void *src); +static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, + const void *src); +static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, + const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_string(pb_ostream_t *stream, + const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, + const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_fixed_length_bytes(pb_ostream_t *stream, + const pb_field_t *field, const void *src); + +#ifdef PB_WITHOUT_64BIT +#define pb_int64_t int32_t +#define pb_uint64_t uint32_t + +static bool checkreturn pb_encode_negative_varint(pb_ostream_t *stream, pb_uint64_t value); +#else +#define pb_int64_t int64_t +#define pb_uint64_t uint64_t +#endif + +/* --- Function pointers to field encoders --- + * Order in the array must match pb_action_t LTYPE numbering. + */ +static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { + &pb_enc_varint, + &pb_enc_uvarint, + &pb_enc_svarint, + &pb_enc_fixed32, + &pb_enc_fixed64, + + &pb_enc_bytes, + &pb_enc_string, + &pb_enc_submessage, + NULL, /* extensions */ + &pb_enc_fixed_length_bytes +}; + +/******************************* + * pb_ostream_t implementation * + *******************************/ + +static bool checkreturn buf_write(pb_ostream_t *stream, const pb_byte_t *buf, + size_t count) +{ + size_t i; + pb_byte_t *dest = (pb_byte_t *)stream->state; + + stream->state = dest + count; + + for (i = 0; i < count; i++) + dest[i] = buf[i]; + + return true; +} + +pb_ostream_t pb_ostream_from_buffer(pb_byte_t *buf, size_t bufsize) +{ + pb_ostream_t stream; + +#ifdef PB_BUFFER_ONLY + stream.callback = (void *)1; /* Just a marker value */ +#else + stream.callback = &buf_write; +#endif + stream.state = buf; + stream.max_size = bufsize; + stream.bytes_written = 0; +#ifndef PB_NO_ERRMSG + stream.errmsg = NULL; +#endif + return stream; +} + +bool checkreturn pb_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count) +{ + if (stream->callback != NULL) { + if (stream->bytes_written + count > stream->max_size) + PB_RETURN_ERROR(stream, "stream full"); + +#ifdef PB_BUFFER_ONLY + if (!buf_write(stream, buf, count)) + PB_RETURN_ERROR(stream, "io error"); +#else + if (!stream->callback(stream, buf, count)) + PB_RETURN_ERROR(stream, "io error"); +#endif + } + + stream->bytes_written += count; + return true; +} + +/************************* + * Encode a single field * + *************************/ + +/* Encode a static array. Handles the size calculations and possible packing. */ +static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, + const void *pData, size_t count, pb_encoder_t func) +{ + size_t i; + const void *p; + size_t size; + + if (count == 0) + return true; + + if (PB_ATYPE(field->type) != PB_ATYPE_POINTER && count > field->array_size) + PB_RETURN_ERROR(stream, "array max size exceeded"); + + /* We always pack arrays if the datatype allows it. */ + if (PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE) { + if (!pb_encode_tag(stream, PB_WT_STRING, field->tag)) + return false; + + /* Determine the total size of packed array. */ + if (PB_LTYPE(field->type) == PB_LTYPE_FIXED32) + size = 4 * count; + else if (PB_LTYPE(field->type) == PB_LTYPE_FIXED64) + size = 8 * count; + else { + pb_ostream_t sizestream = PB_OSTREAM_SIZING; + + p = pData; + for (i = 0; i < count; i++) { + if (!func(&sizestream, field, p)) + return false; + p = (const char *)p + field->data_size; + } + size = sizestream.bytes_written; + } + + if (!pb_encode_varint(stream, (pb_uint64_t)size)) + return false; + + if (stream->callback == NULL) + return pb_write(stream, NULL, size); /* Just sizing.. */ + + /* Write the data */ + p = pData; + for (i = 0; i < count; i++) { + if (!func(stream, field, p)) + return false; + p = (const char *)p + field->data_size; + } + } else { + p = pData; + for (i = 0; i < count; i++) { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + /* Normally the data is stored directly in the array entries, but + * for pointer-type string and bytes fields, the array entries are + * actually pointers themselves also. So we have to dereference once + * more to get to the actual data. + */ + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER && + (PB_LTYPE(field->type) == PB_LTYPE_STRING || + PB_LTYPE(field->type) == PB_LTYPE_BYTES)) { + if (!func(stream, field, *(const void * const *)p)) + return false; + } else { + if (!func(stream, field, p)) + return false; + } + p = (const char *)p + field->data_size; + } + } + + return true; +} + +/* In proto3, all fields are optional and are only encoded + * if their value is "non-zero". + * This function implements the check for the zero value. + */ +static bool pb_check_proto3_default_value(const pb_field_t *field, const void *pData) +{ + pb_type_t type = field->type; + const void *pSize = (const char *)pData + field->size_offset; + + if (PB_HTYPE(type) == PB_HTYPE_REQUIRED) + /* Required proto2 fields inside proto3 submessage, pretty rare case */ + return false; + else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + /* Repeated fields inside proto3 submessage: present if count != 0 */ + return *(const pb_size_t *)pSize == 0; + else if (PB_HTYPE(type) == PB_HTYPE_ONEOF) + /* Oneof fields */ + return *(const pb_size_t *)pSize == 0; + else if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL && field->size_offset) + /* Proto2 optional fields inside proto3 submessage */ + return *(const bool *)pSize == false; + + /* Rest is proto3 singular fields */ + + if (PB_ATYPE(type) == PB_ATYPE_STATIC) { + if (PB_LTYPE(type) == PB_LTYPE_BYTES) { + const pb_bytes_array_t *bytes = (const pb_bytes_array_t *)pData; + + return bytes->size == 0; + } else if (PB_LTYPE(type) == PB_LTYPE_STRING) + return *(const char *)pData == '\0'; + else if (PB_LTYPE(type) == PB_LTYPE_FIXED_LENGTH_BYTES) + /* Fixed length bytes is only empty if its length is fixed + * as 0. Which would be pretty strange, but we can check + * it anyway. + */ + return field->data_size == 0; + else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) { + /* Check all fields in the submessage to find if any of them + * are non-zero. The comparison cannot be done byte-per-byte + * because the C struct may contain padding bytes that must + * be skipped. + */ + pb_field_iter_t iter; + + if (pb_field_iter_begin(&iter, (const pb_field_t *)field->ptr, + pb_const_cast(pData))) { + do { + if (!pb_check_proto3_default_value(iter.pos, iter.pData)) + return false; + } while (pb_field_iter_next(&iter)); + } + return true; + } + } + + /* Catch-all branch that does byte-per-byte comparison for zero value. + * + * This is for all pointer fields, and for static PB_LTYPE_VARINT, + * UVARINT, SVARINT, FIXED32, FIXED64, EXTENSION fields, and also + * callback fields. These all have integer or pointer value which + * can be compared with 0. + */ + pb_size_t i; + const char *p = (const char *)pData; + + for (i = 0; i < field->data_size; i++) { + if (p[i] != 0) + return false; + } + + return true; + +} + +/* Encode a field with static or pointer allocation, i.e. one whose data + * is available to the encoder directly. + */ +static bool checkreturn encode_basic_field(pb_ostream_t *stream, + const pb_field_t *field, const void *pData) +{ + pb_encoder_t func; + bool implicit_has; + const void *pSize = &implicit_has; + + func = PB_ENCODERS[PB_LTYPE(field->type)]; + + if (field->size_offset) + /* Static optional, repeated or oneof field */ + pSize = (const char *)pData + field->size_offset; + else if (PB_HTYPE(field->type) == PB_HTYPE_OPTIONAL) + /* Proto3 style field, optional but without explicit has_ field. */ + implicit_has = !pb_check_proto3_default_value(field, pData); + else + /* Required field, always present */ + implicit_has = true; + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { + /* pData is a pointer to the field, which contains pointer to + * the data. If the 2nd pointer is NULL, it is interpreted as if + * the has_field was false. + */ + pData = *(const void * const *)pData; + implicit_has = (pData != NULL); + } + + switch (PB_HTYPE(field->type)) { + case PB_HTYPE_REQUIRED: + if (!pData) + PB_RETURN_ERROR(stream, "missing required field"); + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!func(stream, field, pData)) + return false; + break; + + case PB_HTYPE_OPTIONAL: + if (*(const bool *)pSize) { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!func(stream, field, pData)) + return false; + } + break; + + case PB_HTYPE_REPEATED: { + pb_size_t count; + + if (field->size_offset != 0) + count = *(const pb_size_t *)pSize; + else + count = field->array_size; + + if (!encode_array(stream, field, pData, count, func)) + return false; + break; + } + + case PB_HTYPE_ONEOF: + if (*(const pb_size_t *)pSize == field->tag) { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!func(stream, field, pData)) + return false; + } + break; + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } + + return true; +} + +/* Encode a field with callback semantics. + * This means that a user function is + * called to provide and encode the actual data. + */ +static bool checkreturn encode_callback_field(pb_ostream_t *stream, + const pb_field_t *field, + const void *pData) +{ + const pb_callback_t *callback = (const pb_callback_t *)pData; + +#ifdef PB_OLD_CALLBACK_STYLE + const void *arg = callback->arg; +#else + void * const *arg = &(callback->arg); +#endif + + if (callback->funcs.encode != NULL) { + if (!callback->funcs.encode(stream, field, arg)) + PB_RETURN_ERROR(stream, "callback error"); + } + + return true; +} + +/* Encode a single field of any callback or static type. */ +static bool checkreturn encode_field(pb_ostream_t *stream, + const pb_field_t *field, const void *pData) +{ + switch (PB_ATYPE(field->type)) { + case PB_ATYPE_STATIC: + case PB_ATYPE_POINTER: + return encode_basic_field(stream, field, pData); + + case PB_ATYPE_CALLBACK: + return encode_callback_field(stream, field, pData); + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +} + +/* Default handler for extension fields. Expects to have a pb_field_t + * pointer in the extension->type->arg field. + */ +static bool checkreturn default_extension_encoder(pb_ostream_t *stream, + const pb_extension_t *extension) +{ + const pb_field_t *field = (const pb_field_t *)extension->type->arg; + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { + /* For pointer extensions, the pointer is stored directly + * in the extension structure. This avoids having an extra + * indirection. + */ + return encode_field(stream, field, &extension->dest); + } else + return encode_field(stream, field, extension->dest); +} + +/* Walk through all the registered extensions and give them a chance + * to encode themselves. + */ +static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, + const void *pData) +{ + const pb_extension_t *extension = *(const pb_extension_t * const *)pData; + + PB_UNUSED(field); + while (extension) { + bool status; + + if (extension->type->encode) + status = extension->type->encode(stream, extension); + else + status = default_extension_encoder(stream, extension); + + if (!status) + return false; + + extension = extension->next; + } + + return true; +} + +/********************* + * Encode all fields * + *********************/ + +static void *pb_const_cast(const void *p) +{ + /* Note: this casts away const, + * in order to use the common field iterator + * logic for both encoding and decoding. + */ + union { + void *p1; + const void *p2; + } t; + + t.p2 = p; + return t.p1; +} + +bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct) +{ + pb_field_iter_t iter; + + if (!pb_field_iter_begin(&iter, fields, pb_const_cast(src_struct))) + return true; /* Empty message type */ + + do { + if (PB_LTYPE(iter.pos->type) == PB_LTYPE_EXTENSION) { + /* Special case for the extension field placeholder */ + if (!encode_extension_field(stream, iter.pos, iter.pData)) + return false; + } else { + /* Regular field */ + if (!encode_field(stream, iter.pos, iter.pData)) + return false; + } + } while (pb_field_iter_next(&iter)); + + return true; +} + +bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct) +{ + return pb_encode_submessage(stream, fields, src_struct); +} + +bool pb_encode_nullterminated(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct) +{ + const pb_byte_t zero = 0; + + if (!pb_encode(stream, fields, src_struct)) + return false; + + return pb_write(stream, &zero, 1); +} + +bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], + const void *src_struct) +{ + pb_ostream_t stream = PB_OSTREAM_SIZING; + + if (!pb_encode(&stream, fields, src_struct)) + return false; + + *size = stream.bytes_written; + return true; +} + +/******************** + * Helper functions * + ********************/ + +#ifdef PB_WITHOUT_64BIT +bool checkreturn pb_encode_negative_varint(pb_ostream_t *stream, pb_uint64_t value) +{ + pb_byte_t buffer[10]; + size_t i = 0; + size_t compensation = 32;/* we need to compensate 32 bits all set to 1 */ + + while (value) { + buffer[i] = (pb_byte_t)((value & 0x7F) | 0x80); + value >>= 7; + if (compensation) { + /* re-set all the compensation bits we can or need */ + size_t bits = compensation > 7 ? 7 : compensation; + + value ^= (pb_uint64_t)((0xFFu >> (8 - bits)) << 25); + /* set the number of bits needed on the lowest + * of the most significant 7 bits + */ + compensation -= bits; + } + i++; + } + buffer[i - 1] &= 0x7F; /* Unset top bit on last byte */ + + return pb_write(stream, buffer, i); +} +#endif + +bool checkreturn pb_encode_varint(pb_ostream_t *stream, pb_uint64_t value) +{ + pb_byte_t buffer[10]; + size_t i = 0; + + if (value <= 0x7F) { + pb_byte_t v = (pb_byte_t)value; + + return pb_write(stream, &v, 1); + } + + while (value) { + buffer[i] = (pb_byte_t)((value & 0x7F) | 0x80); + value >>= 7; + i++; + } + buffer[i-1] &= 0x7F; /* Unset top bit on last byte */ + + return pb_write(stream, buffer, i); +} + +bool checkreturn pb_encode_svarint(pb_ostream_t *stream, pb_int64_t value) +{ + pb_uint64_t zigzagged; + + if (value < 0) + zigzagged = ~((pb_uint64_t)value << 1); + else + zigzagged = (pb_uint64_t)value << 1; + + return pb_encode_varint(stream, zigzagged); +} + +bool checkreturn pb_encode_fixed32(pb_ostream_t *stream, const void *value) +{ + uint32_t val = *(const uint32_t *)value; + pb_byte_t bytes[4]; + + bytes[0] = (pb_byte_t)(val & 0xFF); + bytes[1] = (pb_byte_t)((val >> 8) & 0xFF); + bytes[2] = (pb_byte_t)((val >> 16) & 0xFF); + bytes[3] = (pb_byte_t)((val >> 24) & 0xFF); + + return pb_write(stream, bytes, 4); +} + +#ifndef PB_WITHOUT_64BIT +bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) +{ + uint64_t val = *(const uint64_t *)value; + pb_byte_t bytes[8]; + + bytes[0] = (pb_byte_t)(val & 0xFF); + bytes[1] = (pb_byte_t)((val >> 8) & 0xFF); + bytes[2] = (pb_byte_t)((val >> 16) & 0xFF); + bytes[3] = (pb_byte_t)((val >> 24) & 0xFF); + bytes[4] = (pb_byte_t)((val >> 32) & 0xFF); + bytes[5] = (pb_byte_t)((val >> 40) & 0xFF); + bytes[6] = (pb_byte_t)((val >> 48) & 0xFF); + bytes[7] = (pb_byte_t)((val >> 56) & 0xFF); + + return pb_write(stream, bytes, 8); +} +#endif + +bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, + uint32_t field_number) +{ + pb_uint64_t tag = ((pb_uint64_t)field_number << 3) | wiretype; + + return pb_encode_varint(stream, tag); +} + +bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field) +{ + pb_wire_type_t wiretype; + + switch (PB_LTYPE(field->type)) { + case PB_LTYPE_VARINT: + case PB_LTYPE_UVARINT: + case PB_LTYPE_SVARINT: + wiretype = PB_WT_VARINT; + break; + + case PB_LTYPE_FIXED32: + wiretype = PB_WT_32BIT; + break; + + case PB_LTYPE_FIXED64: + wiretype = PB_WT_64BIT; + break; + + case PB_LTYPE_BYTES: + case PB_LTYPE_STRING: + case PB_LTYPE_SUBMESSAGE: + case PB_LTYPE_FIXED_LENGTH_BYTES: + wiretype = PB_WT_STRING; + break; + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } + + return pb_encode_tag(stream, wiretype, field->tag); +} + +bool checkreturn pb_encode_string(pb_ostream_t *stream, const pb_byte_t *buffer, size_t size) +{ + if (!pb_encode_varint(stream, (pb_uint64_t)size)) + return false; + + return pb_write(stream, buffer, size); +} + +bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct) +{ + /* First calculate the message size using a non-writing substream. */ + pb_ostream_t substream = PB_OSTREAM_SIZING; + size_t size; + bool status; + + if (!pb_encode(&substream, fields, src_struct)) { +#ifndef PB_NO_ERRMSG + stream->errmsg = substream.errmsg; +#endif + return false; + } + + size = substream.bytes_written; + + if (!pb_encode_varint(stream, (pb_uint64_t)size)) + return false; + + if (stream->callback == NULL) + return pb_write(stream, NULL, size); /* Just sizing */ + + if (stream->bytes_written + size > stream->max_size) + PB_RETURN_ERROR(stream, "stream full"); + + /* Use a substream to verify that a callback doesn't write more than + * what it did the first time. + */ + substream.callback = stream->callback; + substream.state = stream->state; + substream.max_size = size; + substream.bytes_written = 0; +#ifndef PB_NO_ERRMSG + substream.errmsg = NULL; +#endif + + status = pb_encode(&substream, fields, src_struct); + + stream->bytes_written += substream.bytes_written; + stream->state = substream.state; +#ifndef PB_NO_ERRMSG + stream->errmsg = substream.errmsg; +#endif + + if (substream.bytes_written != size) + PB_RETURN_ERROR(stream, "submsg size changed"); + + return status; +} + +/* Field encoders */ + +static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + pb_int64_t value = 0; + + if (field->data_size == sizeof(int_least8_t)) + value = *(const int_least8_t *)src; + else if (field->data_size == sizeof(int_least16_t)) + value = *(const int_least16_t *)src; + else if (field->data_size == sizeof(int32_t)) + value = *(const int32_t *)src; + else if (field->data_size == sizeof(pb_int64_t)) + value = *(const pb_int64_t *)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + +#ifdef PB_WITHOUT_64BIT + if (value < 0) + return pb_encode_negative_varint(stream, (pb_uint64_t)value); +#endif + return pb_encode_varint(stream, (pb_uint64_t)value); +} + +static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + pb_uint64_t value = 0; + + if (field->data_size == sizeof(uint_least8_t)) + value = *(const uint_least8_t *)src; + else if (field->data_size == sizeof(uint_least16_t)) + value = *(const uint_least16_t *)src; + else if (field->data_size == sizeof(uint32_t)) + value = *(const uint32_t *)src; + else if (field->data_size == sizeof(pb_uint64_t)) + value = *(const pb_uint64_t *)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + return pb_encode_varint(stream, value); +} + +static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + pb_int64_t value = 0; + + if (field->data_size == sizeof(int_least8_t)) + value = *(const int_least8_t *)src; + else if (field->data_size == sizeof(int_least16_t)) + value = *(const int_least16_t *)src; + else if (field->data_size == sizeof(int32_t)) + value = *(const int32_t *)src; + else if (field->data_size == sizeof(pb_int64_t)) + value = *(const pb_int64_t *)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + return pb_encode_svarint(stream, value); +} + +static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + PB_UNUSED(field); +#ifndef PB_WITHOUT_64BIT + return pb_encode_fixed64(stream, src); +#else + PB_UNUSED(src); + PB_RETURN_ERROR(stream, "no 64bit support"); +#endif +} + +static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + PB_UNUSED(field); + return pb_encode_fixed32(stream, src); +} + +static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, + const void *src) +{ + const pb_bytes_array_t *bytes = NULL; + + bytes = (const pb_bytes_array_t *)src; + + if (src == NULL) { + /* Treat null pointer as an empty bytes field */ + return pb_encode_string(stream, NULL, 0); + } + + if (PB_ATYPE(field->type) == PB_ATYPE_STATIC && + PB_BYTES_ARRAY_T_ALLOCSIZE(bytes->size) > field->data_size) + PB_RETURN_ERROR(stream, "bytes size exceeded"); + + return pb_encode_string(stream, bytes->bytes, bytes->size); +} + +static bool checkreturn pb_enc_string(pb_ostream_t *stream, + const pb_field_t *field, + const void *src) +{ + size_t size = 0; + size_t max_size = field->data_size; + const char *p = (const char *)src; + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + max_size = (size_t)-1; + + if (src == NULL) + size = 0; /* Treat null pointer as an empty string */ + else { + /* strnlen() is not always available, so just use a loop */ + while (size < max_size && *p != '\0') { + size++; + p++; + } + } + + return pb_encode_string(stream, (const pb_byte_t *)src, size); +} + +static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, + const pb_field_t *field, + const void *src) +{ + if (field->ptr == NULL) + PB_RETURN_ERROR(stream, "invalid field descriptor"); + + return pb_encode_submessage(stream, (const pb_field_t *)field->ptr, src); +} + +static bool checkreturn pb_enc_fixed_length_bytes(pb_ostream_t *stream, + const pb_field_t *field, + const void *src) +{ + return pb_encode_string(stream, (const pb_byte_t *)src, field->data_size); +} + diff --git a/drivers/misc/vehicle/nanopb-c/pb_encode.h b/drivers/misc/vehicle/nanopb-c/pb_encode.h new file mode 100644 index 000000000000..0ad04e494304 --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_encode.h @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2011 Petteri Aimonen + * + * This software is provided 'as-is', without any express or + * implied warranty. In no event will the authors be held liable + * for any damages arising from the use of this software. + * + * Permission is granted to anyone to use this software for any + * purpose, including commercial applications, and to alter it and + * redistribute it freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you + * must not claim that you wrote the original software. If you use + * this software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and + * must not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ +/* pb_encode.h: Functions to encode protocol buffers. Depends on pb_encode.c. + * The main function is pb_encode. You also need an output stream, and the + * field descriptions created by nanopb_generator.py. + */ + +#ifndef PB_ENCODE_H_INCLUDED +#define PB_ENCODE_H_INCLUDED + +#include "pb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Structure for defining custom output streams. You will need to provide + * a callback function to write the bytes to your storage, which can be + * for example a file or a network socket. + * + * The callback must conform to these rules: + * + * 1) Return false on IO errors. This will cause encoding to abort. + * 2) You can use state to store your own data (e.g. buffer pointer). + * 3) pb_write will update bytes_written after your callback runs. + * 4) Substreams will modify max_size and bytes_written. Don't use them + * to calculate any pointers. + */ +struct pb_ostream_s { +#ifdef PB_BUFFER_ONLY + /* Callback pointer is not used in buffer-only configuration. + * Having an int pointer here allows binary compatibility but + * gives an error if someone tries to assign callback function. + * Also, NULL pointer marks a 'sizing stream' that does not + * write anything. + */ + int *callback; +#else + bool (*callback)(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); +#endif + void *state; /* Free field for use by callback implementation. */ + size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ + size_t bytes_written; /* Number of bytes written so far. */ + +#ifndef PB_NO_ERRMSG + const char *errmsg; +#endif +}; + +/*************************** + * Main encoding functions * + ***************************/ + +/* Encode a single protocol buffers message from C structure into a stream. + * Returns true on success, false on any failure. + * The actual struct pointed to by src_struct must match the description in fields. + * All required fields in the struct are assumed to have been filled in. + * + * Example usage: + * MyMessage msg = {}; + * uint8_t buffer[64]; + * pb_ostream_t stream; + * + * msg.field1 = 42; + * stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + * pb_encode(&stream, MyMessage_fields, &msg); + */ +bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); + +/* Same as pb_encode, but prepends the length of the message as a varint. + * Corresponds to writeDelimitedTo() in Google's protobuf API. + */ +bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct); + +/* Same as pb_encode, but appends a null byte to the message for termination. + * NOTE: This behaviour is not supported in most other protobuf implementations, + * so pb_encode_delimited() + * is a better option for compatibility. + */ +bool pb_encode_nullterminated(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct); + +/* Encode the message to get the size of the encoded data, but do not store + * the data. + */ +bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], + const void *src_struct); + +/************************************** + * Functions for manipulating streams * + **************************************/ + +/* Create an output stream for writing into a memory buffer. + * The number of bytes written can be found in stream.bytes_written after + * encoding the message. + * + * Alternatively, you can use a custom stream that writes directly to e.g. + * a file or a network socket. + */ +pb_ostream_t pb_ostream_from_buffer(pb_byte_t *buf, size_t bufsize); + +/* Pseudo-stream for measuring the size of a message without actually storing + * the encoded data. + * + * Example usage: + * MyMessage msg = {}; + * pb_ostream_t stream = PB_OSTREAM_SIZING; + * pb_encode(&stream, MyMessage_fields, &msg); + * printf("Message size is %d\n", stream.bytes_written); + */ +#ifndef PB_NO_ERRMSG +#define PB_OSTREAM_SIZING {0, 0, 0, 0, 0} +#else +#define PB_OSTREAM_SIZING {0, 0, 0, 0} +#endif + +/* Function to write into a pb_ostream_t stream. You can use this if you need + * to append or prepend some custom headers to the message. + */ +bool pb_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); + + +/************************************************ + * Helper functions for writing field callbacks * + ************************************************/ + +/* Encode field header based on type and field number defined in the field + * structure. Call this from the callback before writing out field contents. + */ +bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field); + +/* Encode field header by manually specifying wire type. You need to use this + * if you want to write out packed arrays from a callback field. + */ +bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, + uint32_t field_number); + +/* Encode an integer in the varint format. + * This works for bool, enum, int32, int64, uint32 and uint64 field types. + */ +#ifndef PB_WITHOUT_64BIT +bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); +#else +bool pb_encode_varint(pb_ostream_t *stream, uint32_t value); +#endif + +/* Encode an integer in the zig-zagged svarint format. + * This works for sint32 and sint64. + */ +#ifndef PB_WITHOUT_64BIT +bool pb_encode_svarint(pb_ostream_t *stream, int64_t value); +#else +bool pb_encode_svarint(pb_ostream_t *stream, int32_t value); +#endif + +/* Encode a string or bytes type field. For strings, pass strlen(s) as size. */ +bool pb_encode_string(pb_ostream_t *stream, const pb_byte_t *buffer, size_t size); + +/* Encode a fixed32, sfixed32 or float value. + * You need to pass a pointer to a 4-byte wide C variable. + */ +bool pb_encode_fixed32(pb_ostream_t *stream, const void *value); + +#ifndef PB_WITHOUT_64BIT +/* Encode a fixed64, sfixed64 or double value. + * You need to pass a pointer to a 8-byte wide C variable. + */ +bool pb_encode_fixed64(pb_ostream_t *stream, const void *value); +#endif + +/* Encode a submessage field. + * You need to pass the pb_field_t array and pointer to struct, just like + * with pb_encode(). This internally encodes the submessage twice, first to + * calculate message size and then to actually write it out. + */ +bool pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], + const void *src_struct); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/drivers/misc/vehicle/nanopb-c/pb_system.h b/drivers/misc/vehicle/nanopb-c/pb_system.h new file mode 100644 index 000000000000..2166e4b7256b --- /dev/null +++ b/drivers/misc/vehicle/nanopb-c/pb_system.h @@ -0,0 +1,27 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef PB_SYSTEM_H +#define PB_SYSTEM_H + +#include + +typedef u8 uint_least8_t; +typedef s8 int_least8_t; +typedef u8 uint_fast8_t; + +typedef u16 uint_least16_t; +typedef s16 int_least16_t; + +#endif diff --git a/drivers/misc/vehicle/vehicle-adc.c b/drivers/misc/vehicle/vehicle-adc.c new file mode 100644 index 000000000000..0477f52ac061 --- /dev/null +++ b/drivers/misc/vehicle/vehicle-adc.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * vehicle-adc.c -- I2C register interface access for adc + * + * Copyright (c) 2025-2030 Rockchip Electronics Co., Ltd. + * + * Author: LuoWei + */ + +#include "core.h" + +static int vehicle_adc_init_channel(struct vehicle *vehicle, struct iio_channel **channel, + const char *name) +{ + int err = 0; + enum iio_chan_type type; + struct device *dev = vehicle->vehicle_adc->dev; + + struct iio_channel *tmp_channel = devm_iio_channel_get(dev, name); + + if (IS_ERR(tmp_channel)) + return PTR_ERR(tmp_channel); + + if (!tmp_channel->indio_dev) + return -ENXIO; + + err = iio_get_channel_type(tmp_channel, &type); + if (err < 0) + return err; + + if (type != IIO_VOLTAGE) { + dev_err(dev, "Incompatible channel type %d\n", type); + return -EINVAL; + } + + *channel = tmp_channel; + + return err; +} + +static int vehicle_adc_update_data(struct vehicle *vehicle) +{ + int val; + struct device *dev = vehicle->vehicle_adc->dev; + + vehicle->vehicle_data.turn = TURN_0; + if (vehicle->vehicle_adc->gear_adc_chn) { + if (iio_read_channel_raw(vehicle->vehicle_adc->gear_adc_chn, &val) < 0) + dev_err(dev, "Failed to read gear adc value\n"); + + if (val < 200) + vehicle->vehicle_data.gear = GEAR_0; + else if (val > 200 && val < 1200) + vehicle->vehicle_data.gear = GEAR_1; + else if (val > 1200 && val < 2200) + vehicle->vehicle_data.gear = GEAR_3; + else if (val > 2200 && val < 3200) + vehicle->vehicle_data.gear = GEAR_2; + } + + if (vehicle->vehicle_adc->turn_left_adc_chn) { + if (iio_read_channel_raw(vehicle->vehicle_adc->turn_left_adc_chn, &val) < 0) + dev_err(dev, "Failed to read turn adc value\n"); + + if (val < 100) + vehicle->vehicle_data.turn = TURN_1; + } + + if (vehicle->vehicle_adc->turn_right_adc_chn) { + if (iio_read_channel_raw(vehicle->vehicle_adc->turn_right_adc_chn, &val) < 0) + dev_err(dev, "Failed to read turn adc value\n"); + + if (val < 100) + vehicle->vehicle_data.turn = TURN_2; + } + + vehicle_set_property(VEHICLE_GEAR, 0, vehicle->vehicle_data.gear, 0); + vehicle_set_property(VEHICLE_TURN_SIGNAL, 0, vehicle->vehicle_data.turn, 0); + + dev_info(dev, "gear %u turn %u\n", vehicle->vehicle_data.gear, + vehicle->vehicle_data.turn); + + /* to do others adc */ + return 0; +} + +static void vehicle_adc_delay_work_func(struct work_struct *work) +{ + struct vehicle_adc *vehicle_adc = container_of(work, struct vehicle_adc, + vehicle_delay_work.work); + struct device *dev = vehicle_adc->dev; + + vehicle_adc_update_data(g_vehicle_hw); + + if (vehicle_adc->use_delay_work) + queue_delayed_work(vehicle_adc->vehicle_wq, &vehicle_adc->vehicle_delay_work, + msecs_to_jiffies(1000)); + + dev_info(dev, "%s\n", __func__); +} + +static irqreturn_t vehicle_adc_irq_handle(int irq, void *_data) +{ + struct vehicle_adc *vehicle_adc = _data; + + queue_delayed_work(vehicle_adc->vehicle_wq, &vehicle_adc->vehicle_delay_work, + msecs_to_jiffies(10)); + + return IRQ_HANDLED; +} + +static int vehicle_adc_irq_init(struct vehicle_adc *vehicle_adc) +{ + struct device *dev = vehicle_adc->dev; + int ret; + + vehicle_adc->irq = platform_get_irq(vehicle_adc->pdev, 0); + if (vehicle_adc->irq < 0) { + dev_err(dev, "failed to get irq\n"); + return -ENODEV; + } + ret = devm_request_threaded_irq(dev, vehicle_adc->irq, + vehicle_adc_irq_handle, NULL, + IRQF_TRIGGER_HIGH, + dev_name(dev), vehicle_adc); + + if (ret < 0) { + dev_err(dev, "error: irq %d\n", vehicle_adc->irq); + return ret; + } + + return 0; +} + +static int adc_hw_init(struct vehicle *vehicle) +{ + int err = 0; + struct device *dev = vehicle->dev; + + err = vehicle_adc_init_channel(vehicle, + &vehicle->vehicle_adc->gear_adc_chn, + "gear"); + if (err) + dev_err(dev, "failed to get turn adc channel.\n"); + + err = vehicle_adc_init_channel(vehicle, + &vehicle->vehicle_adc->turn_left_adc_chn, + "turn_left"); + if (err) + dev_err(dev, "failed to get turn_left adc channel.\n"); + + err = vehicle_adc_init_channel(vehicle, + &vehicle->vehicle_adc->turn_right_adc_chn, + "turn_right"); + if (err) + dev_err(dev, "failed to get turn_right adc channel.\n"); + + vehicle->vehicle_adc->vehicle_wq = alloc_ordered_workqueue("%s", + WQ_MEM_RECLAIM | WQ_FREEZABLE, + "vehicle-adc-wq"); + mutex_init(&vehicle->vehicle_adc->wq_lock); + INIT_DELAYED_WORK(&vehicle->vehicle_adc->vehicle_delay_work, vehicle_adc_delay_work_func); + + //vehicle->vehicle_adc->use_delay_work = + // of_property_read_bool(vehicle->dev->of_node, "use-delay-work"); + vehicle->vehicle_adc->use_delay_work = 1; + if (vehicle->vehicle_adc->use_delay_work) { + queue_delayed_work(vehicle->vehicle_adc->vehicle_wq, + &vehicle->vehicle_adc->vehicle_delay_work, + msecs_to_jiffies(100)); + VEHICLE_DBG("%s: vehicle_adc->use_delay_work=%d\n", __func__, + vehicle->vehicle_adc->use_delay_work); + } else { + vehicle_adc_irq_init(vehicle->vehicle_adc); + VEHICLE_DBG("%s: vehicle_adc->use_delay_work=%d\n", __func__, + vehicle->vehicle_adc->use_delay_work); + } + + return 0; +} + +static int adc_pm_suspend(struct vehicle *vehicle) +{ + return 0; +} + +static int adc_pm_resume(struct vehicle *vehicle) +{ + return 0; +} + +struct vehicle_hw_data vehicle_adc_data = { + .name = "vehicle-adc", + .vehicle_hw_type = VEHICLE_HW_TYPE_ADC, + .data_update = vehicle_adc_update_data, + .hw_init = adc_hw_init, + .suspend = adc_pm_suspend, + .resume = adc_pm_resume, +}; +EXPORT_SYMBOL_GPL(vehicle_adc_data); + +static int vehicle_adc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct vehicle_adc *vehicle_adc; + + if (!pdev) + return -ENOMEM; + + if (g_vehicle_hw == NULL) + return -ENOMEM; + + vehicle_adc = devm_kzalloc(dev, sizeof(*vehicle_adc), GFP_KERNEL); + if (vehicle_adc == NULL) + return -ENOMEM; + + platform_set_drvdata(pdev, vehicle_adc); + vehicle_adc->hw_data = &vehicle_adc_data; + vehicle_adc->dev = dev; + vehicle_adc->pdev = pdev; + + g_vehicle_hw->vehicle_adc = vehicle_adc; + + adc_hw_init(g_vehicle_hw); + + pr_info("%s successfully\n", __func__); + + return 0; +} + +static int vehicle_adc_remove(struct platform_device *pdev) +{ + struct vehicle_adc *vehicle_adc = platform_get_drvdata(pdev); + + destroy_workqueue(vehicle_adc->vehicle_wq); + return 0; +} + +static const struct of_device_id vehicle_adc_id[] = { + { .compatible = "rockchip,vehicle-adc", }, + { .compatible = "rockchip,vehicle-dummy-adc", }, + {}, +}; + +static struct platform_driver vehicle_adc_device_driver = { + .probe = vehicle_adc_probe, + .remove = vehicle_adc_remove, + .driver = { + .name = "vehicle-adc", + .of_match_table = vehicle_adc_id, + } +}; + +static int vehicle_adc_init(void) +{ + int err; + + err = platform_driver_register(&vehicle_adc_device_driver); + if (err) + pr_err("Failed to register vehicle driver\n"); + + pr_info("%s successfully\n", __func__); + return err; +} + +static void __exit vehicle_adc_exit(void) +{ + platform_driver_unregister(&vehicle_adc_device_driver); +} + +postcore_initcall(vehicle_adc_init); +module_exit(vehicle_adc_exit); + +MODULE_LICENSE("GPL"); diff --git a/drivers/misc/vehicle/vehicle-gpio.c b/drivers/misc/vehicle/vehicle-gpio.c new file mode 100644 index 000000000000..f10ed247e400 --- /dev/null +++ b/drivers/misc/vehicle/vehicle-gpio.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * vehicle-gpio.c -- I2C register interface access for gpio + * + * Copyright (c) 2025-2030 Rockchip Electronics Co., Ltd. + * + * Author: LuoWei + */ + +#include "core.h" + +static int vehicle_gpio_init_io(struct vehicle_gpio *vehicle_gpio) +{ + int err = 0; + struct device *dev = vehicle_gpio->dev; + + vehicle_gpio->gear_gpio_reverse = devm_gpiod_get_optional(dev, "reverse", GPIOD_IN); + if (IS_ERR(vehicle_gpio->gear_gpio_reverse)) { + dev_err(dev, "failed to get gpio reverse\n"); + err = PTR_ERR(vehicle_gpio->gear_gpio_reverse); + } + + vehicle_gpio->gear_gpio_park = devm_gpiod_get_optional(dev, "park", GPIOD_IN); + if (IS_ERR(vehicle_gpio->gear_gpio_park)) { + dev_err(dev, "failed to get gpio park\n"); + err = PTR_ERR(vehicle_gpio->gear_gpio_park); + } + + return err; +} + +static int vehicle_gpio_update_data(struct vehicle *vehicle) +{ + int gear = 0; + int park_value = 0; + int reverse_value = 0; + struct device *dev = vehicle->vehicle_gpio->dev; + + if (vehicle->vehicle_gpio->gear_gpio_park) + park_value = !!gpiod_get_value(vehicle->vehicle_gpio->gear_gpio_park); + + if (vehicle->vehicle_gpio->gear_gpio_reverse) + reverse_value = !!gpiod_get_value(vehicle->vehicle_gpio->gear_gpio_reverse); + + dev_info(dev, "vehicle gpio %d %d\n", park_value, reverse_value); + + if (park_value && reverse_value) + vehicle->vehicle_data.gear = GEAR_2; + else if (!park_value && reverse_value) + vehicle->vehicle_data.gear = GEAR_1; + else if (park_value && !reverse_value) + vehicle->vehicle_data.gear = GEAR_0; + else + vehicle->vehicle_data.gear = GEAR_3; + + vehicle_set_property(VEHICLE_GEAR, 0, vehicle->vehicle_data.gear, 0); + vehicle_set_property(VEHICLE_TURN_SIGNAL, 0, vehicle->vehicle_data.turn, 0); + + dev_info(dev, "gear %u turn %u\n", vehicle->vehicle_data.gear, + vehicle->vehicle_data.turn); + + /* to do others gpio*/ + + return gear; +} + +static void vehicle_gpio_delay_work_func(struct work_struct *work) +{ + struct vehicle_gpio *vehicle_gpio = container_of(work, struct vehicle_gpio, + vehicle_delay_work.work); + struct device *dev = vehicle_gpio->dev; + + vehicle_gpio_update_data(g_vehicle_hw); + + if (vehicle_gpio->use_delay_work) + queue_delayed_work(vehicle_gpio->vehicle_wq, &vehicle_gpio->vehicle_delay_work, + msecs_to_jiffies(1000)); + + dev_info(dev, "%s\n", __func__); +} + +static irqreturn_t vehicle_gpio_irq_handle(int irq, void *_data) +{ + struct vehicle_gpio *vehicle_gpio = _data; + + queue_delayed_work(vehicle_gpio->vehicle_wq, &vehicle_gpio->vehicle_delay_work, + msecs_to_jiffies(10)); + + return IRQ_HANDLED; +} + +static int vehicle_gpio_irq_init(struct vehicle_gpio *vehicle_gpio) +{ + struct device *dev = vehicle_gpio->dev; + int ret; + + vehicle_gpio->irq = platform_get_irq(vehicle_gpio->pdev, 0); + if (vehicle_gpio->irq < 0) { + dev_err(dev, "failed to get irq\n"); + return -ENODEV; + } + ret = devm_request_threaded_irq(dev, vehicle_gpio->irq, + vehicle_gpio_irq_handle, NULL, + IRQF_TRIGGER_HIGH, + dev_name(dev), vehicle_gpio); + + if (ret < 0) { + dev_err(dev, "error: irq %d\n", vehicle_gpio->irq); + return ret; + } + + return 0; +} + +static int gpio_hw_init(struct vehicle *vehicle) +{ + vehicle_gpio_init_io(vehicle->vehicle_gpio); + vehicle->vehicle_gpio->vehicle_wq = + alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM | WQ_FREEZABLE, + "vehicle-gpio-wq"); + mutex_init(&vehicle->vehicle_gpio->wq_lock); + INIT_DELAYED_WORK(&vehicle->vehicle_gpio->vehicle_delay_work, vehicle_gpio_delay_work_func); + + vehicle->vehicle_gpio->use_delay_work = + of_property_read_bool(vehicle->vehicle_gpio->dev->of_node, "use-delay-work"); + if (vehicle->vehicle_gpio->use_delay_work) { + queue_delayed_work(vehicle->vehicle_gpio->vehicle_wq, + &vehicle->vehicle_gpio->vehicle_delay_work, + msecs_to_jiffies(100)); + VEHICLE_DBG("%s: vehicle_gpio->use_delay_work=%d\n", __func__, + vehicle->vehicle_gpio->use_delay_work); + } else { + vehicle_gpio_irq_init(vehicle->vehicle_gpio); + VEHICLE_DBG("%s: vehicle_gpio->use_delay_work=%d\n", __func__, + vehicle->vehicle_gpio->use_delay_work); + } + + return 0; +} + +static int gpio_pm_suspend(struct vehicle *vehicle) +{ + return 0; +} + +static int gpio_pm_resume(struct vehicle *vehicle) +{ + return 0; +} + +struct vehicle_hw_data vehicle_gpio_data = { + .name = "vehicle-gpio", + .vehicle_hw_type = VEHICLE_HW_TYPE_GPIO, + .data_update = vehicle_gpio_update_data, + .hw_init = gpio_hw_init, + .suspend = gpio_pm_suspend, + .resume = gpio_pm_resume, +}; +EXPORT_SYMBOL_GPL(vehicle_gpio_data); + +static int vehicle_gpio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct vehicle_gpio *vehicle_gpio; + + if (!pdev) + return -ENOMEM; + + if (g_vehicle_hw == NULL) + return -ENOMEM; + + vehicle_gpio = devm_kzalloc(dev, sizeof(*vehicle_gpio), GFP_KERNEL); + if (vehicle_gpio == NULL) + return -ENOMEM; + + platform_set_drvdata(pdev, vehicle_gpio); + vehicle_gpio->hw_data = &vehicle_gpio_data; + vehicle_gpio->dev = dev; + vehicle_gpio->pdev = pdev; + + g_vehicle_hw->vehicle_gpio = vehicle_gpio; + + gpio_hw_init(g_vehicle_hw); + + return 0; +} + +static int vehicle_gpio_remove(struct platform_device *pdev) +{ + struct vehicle_gpio *vehicle_gpio = platform_get_drvdata(pdev); + + destroy_workqueue(vehicle_gpio->vehicle_wq); + return 0; +} + +static const struct of_device_id vehicle_gpio_id[] = { + { .compatible = "rockchip,vehicle-gpio", }, + { .compatible = "rockchip,vehicle-dummy-gpio", }, + {}, +}; + +static struct platform_driver vehicle_gpio_device_driver = { + .probe = vehicle_gpio_probe, + .remove = vehicle_gpio_remove, + .driver = { + .name = "vehicle-gpio", + .of_match_table = vehicle_gpio_id, + } +}; + +static int vehicle_gpio_init(void) +{ + int err; + + err = platform_driver_register(&vehicle_gpio_device_driver); + if (err) + pr_err("Failed to register vehicle driver\n"); + + return err; +} + +static void __exit vehicle_gpio_exit(void) +{ + platform_driver_unregister(&vehicle_gpio_device_driver); +} + +postcore_initcall(vehicle_gpio_init); +module_exit(vehicle_gpio_exit); + +MODULE_LICENSE("GPL"); diff --git a/drivers/misc/vehicle/vehicle-spi.c b/drivers/misc/vehicle/vehicle-spi.c new file mode 100644 index 000000000000..1aca58977e64 --- /dev/null +++ b/drivers/misc/vehicle/vehicle-spi.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * vehicle-spi.c -- I2C register interface access for spi + * + * Copyright (c) 2025-2030 Rockchip Electronics Co., Ltd. + * + * Author: LuoWei + */ + +#include "core.h" +#ifdef CONFIG_VEHICLE_SPI_PROTOCOL +#include "vehicle_spi_protocol.h" +#endif + +static struct completion spi_complete; +#define SPI_TIMEOUT_MS 20 + +static void spi_complete_callback(void *arg) +{ + complete(&spi_complete); +} + +int vehicle_spi_write_slt(struct vehicle *vehicle, const void *txbuf, size_t n) +{ + int ret = -1; + struct spi_device *spi = NULL; + struct spi_transfer t = { + .tx_buf = txbuf, + .len = n, + .bits_per_word = 8, + }; + struct spi_message m; + + spi = vehicle->vehicle_spi->spi; + reinit_completion(&spi_complete); + spi_message_init(&m); + spi_message_add_tail(&t, &m); + m.complete = spi_complete_callback; + ret = spi_async(spi, &m); + + if (ret) { + dev_err(&spi->dev, "SPI write async error: %d\n", ret); + return ret; + } + + if (!wait_for_completion_timeout(&spi_complete, msecs_to_jiffies(SPI_TIMEOUT_MS))) { + dev_err(&spi->dev, "SPI write operation timed out\n"); + return -ETIMEDOUT; + } + + return ret; +} + +int vehicle_spi_read_slt(struct vehicle *vehicle, void *rxbuf, size_t n) +{ + int ret = -1; + struct spi_device *spi = NULL; + struct spi_transfer t = { + .rx_buf = rxbuf, + .len = n, + .bits_per_word = 8, + }; + struct spi_message m; + + spi = vehicle->vehicle_spi->spi; + reinit_completion(&spi_complete); + spi_message_init(&m); + spi_message_add_tail(&t, &m); + m.complete = spi_complete_callback; + ret = spi_async(spi, &m); + if (ret) { + dev_err(&spi->dev, "SPI read async error: %d\n", ret); + return ret; + } + + if (!wait_for_completion_timeout(&spi_complete, msecs_to_jiffies(SPI_TIMEOUT_MS))) { + dev_err(&spi->dev, "SPI read operation timed out\n"); + return -ETIMEDOUT; + } + + return ret; +} + +static int vehicle_spi_update_data(struct vehicle *vehicle) +{ + int i = 0; + unsigned int times = 1, size = 12; + unsigned long us = 0, bytes = 0; + unsigned char *rxbuf = NULL; + ktime_t start_time; + ktime_t end_time; + ktime_t cost_time; + struct device *dev = vehicle->vehicle_spi->dev; + + rxbuf = kzalloc(size, GFP_KERNEL); + + if (!rxbuf) + return -ENOMEM; + + start_time = ktime_get(); + for (i = 0; i < times; i++) +#ifndef CONFIG_VEHICLE_SPI_PROTOCOL + vehicle_spi_read_slt(vehicle, rxbuf, size); +#else + Analyze_read_data(vehicle, rxbuf, size); +#endif + end_time = ktime_get(); + cost_time = ktime_sub(end_time, start_time); + us = ktime_to_us(cost_time); + + bytes = size * times * 1; + bytes = bytes * 1000 / us; + pr_info("spi read %d*%d cost %ldus speed:%ldKB/S\n", size, times, us, bytes); + print_hex_dump(KERN_ERR, "SPI RX: ", + DUMP_PREFIX_OFFSET, + 16, + 1, + rxbuf, + size, + 1); + + kfree(rxbuf); + vehicle_set_property(VEHICLE_GEAR, 0, vehicle->vehicle_data.gear, 0); + vehicle_set_property(VEHICLE_TURN_SIGNAL, 0, vehicle->vehicle_data.turn, 0); + + dev_info(dev, "gear %u turn %u\n", vehicle->vehicle_data.gear, vehicle->vehicle_data.turn); + + /* to do others spi */ + return 0; +} + + +static void vehicle_spi_delay_work_func(struct work_struct *work) +{ + struct vehicle_spi *vehicle_spi = container_of(work, struct vehicle_spi, + vehicle_delay_work.work); + struct device *dev = vehicle_spi->dev; + + vehicle_spi_update_data(g_vehicle_hw); + + if (vehicle_spi->use_delay_work) + queue_delayed_work(vehicle_spi->vehicle_wq, &vehicle_spi->vehicle_delay_work, + msecs_to_jiffies(1000)); + + dev_info(dev, "%s end\n", __func__); +} + +static irqreturn_t vehicle_spi_irq_handle(int irq, void *_data) +{ + struct vehicle_spi *vehicle_spi = _data; + + queue_delayed_work(vehicle_spi->vehicle_wq, &vehicle_spi->vehicle_delay_work, + msecs_to_jiffies(0)); + + return IRQ_HANDLED; +} + +static int vehicle_spi_irq_init(struct vehicle_spi *vehicle_spi) +{ + struct device *dev = vehicle_spi->dev; + struct gpio_desc *irq_gpio_desc; + int ret; + + irq_gpio_desc = devm_gpiod_get_optional(dev, "irq", GPIOD_IN); + if (IS_ERR_OR_NULL(irq_gpio_desc)) + dev_warn(dev, "Failed to request irq-gpio\n"); + else { + vehicle_spi->irq = gpiod_to_irq(irq_gpio_desc); + ret = devm_request_threaded_irq(dev, vehicle_spi->irq, + vehicle_spi_irq_handle, NULL, + IRQF_TRIGGER_FALLING, + dev_name(dev), vehicle_spi); + + if (ret < 0) { + dev_err(dev, "error: irq %d\n", vehicle_spi->irq); + gpio_free(vehicle_spi->irq); + return ret; + } + } + + return 0; +} + +static int spi_hw_init(struct vehicle *vehicle) +{ + vehicle->vehicle_spi->vehicle_wq = alloc_ordered_workqueue("%s", + WQ_MEM_RECLAIM | WQ_FREEZABLE, + "vehicle-spi-wq"); + mutex_init(&vehicle->vehicle_spi->wq_lock); + INIT_DELAYED_WORK(&vehicle->vehicle_spi->vehicle_delay_work, + vehicle_spi_delay_work_func); + + vehicle->vehicle_spi->use_delay_work = + of_property_read_bool(vehicle->vehicle_spi->dev->of_node, "use-delay-work"); + + if (vehicle->vehicle_spi->use_delay_work) { + queue_delayed_work(vehicle->vehicle_spi->vehicle_wq, + &vehicle->vehicle_spi->vehicle_delay_work, + msecs_to_jiffies(100)); + VEHICLE_DBG("%s: vehicle_spi->use_delay_work=%d\n", __func__, + vehicle->vehicle_spi->use_delay_work); + } else { + vehicle_spi_irq_init(vehicle->vehicle_spi); + VEHICLE_DBG("%s: vehicle_spi->use_delay_work=%d\n", __func__, + vehicle->vehicle_spi->use_delay_work); + } + + return 0; +} + +static int spi_pm_suspend(struct vehicle *vehicle) +{ + return 0; +} + +static int spi_pm_resume(struct vehicle *vehicle) +{ + return 0; +} + +struct vehicle_hw_data vehicle_spi_data = { + .name = "vehicle-spi", + .vehicle_hw_type = VEHICLE_HW_TYPE_SPI, + .data_update = vehicle_spi_update_data, + .hw_init = spi_hw_init, + .suspend = spi_pm_suspend, + .resume = spi_pm_resume, +}; +EXPORT_SYMBOL_GPL(vehicle_spi_data); + +static ssize_t spi_test_write(struct file *file, + const char __user *buf, size_t n, loff_t *offset) +{ + int argc = 0, i; + char tmp[64]; + char *argv[16]; + char *cmd, *data; + unsigned int id = 0, times = 0, size = 0, cmd_spi = 0; + unsigned long us = 0, bytes = 0; + char *txbuf = NULL; + ktime_t start_time; + ktime_t end_time; + ktime_t cost_time; + + if (n >= sizeof(tmp)) { + pr_info("%s error size > 64\n", __func__); + return -EINVAL; + } + + memset(tmp, 0, sizeof(tmp)); + if (copy_from_user(tmp, buf, n)) + return -EFAULT; + cmd = tmp; + data = tmp; + + memset(argv, 0, sizeof(argv)); + + while (data < (tmp + n)) { + data = strstr(data, " "); + if (!data) + break; + *data = 0; + argv[argc] = ++data; + argc++; + if (argc >= 16) + break; + } + + tmp[n - 1] = 0; + + if (!strcmp(cmd, "write")) { + if (kstrtoint(argv[0], 10, &id) < 0) + return -EFAULT; + + if (kstrtoint(argv[1], 10, ×) < 0) + return -EFAULT; + + if (kstrtoint(argv[2], 10, &size) < 0) + return -EFAULT; + + if (kstrtoint(argv[3], 16, &cmd_spi) < 0) + return -EFAULT; + + txbuf = kzalloc(size, GFP_KERNEL); + if (!txbuf) + return n; + + for (i = 0; i < size; i++) { + if (kstrtoint(argv[4+i], 16, (int *)(txbuf+i)) < 0) + return -EFAULT; + } + + start_time = ktime_get(); + for (i = 0; i < times; i++) +#ifndef CONFIG_VEHICLE_SPI_PROTOCOL + vehicle_spi_write_slt(g_vehicle_hw, txbuf, size); +#else + Analyze_write_data(g_vehicle_hw, (unsigned char)cmd_spi, txbuf, size); +#endif + end_time = ktime_get(); + cost_time = ktime_sub(end_time, start_time); + us = ktime_to_us(cost_time); + + bytes = size * times * 1; + bytes = bytes * 1000 / us; + pr_info("spi write %d*%d cost %ldus speed:%ldKB/S\n", size, times, us, bytes); + kfree(txbuf); + } + return n; +} +static const struct file_operations spi_test_fops = { + .write = spi_test_write, +}; + +static struct miscdevice spi_test_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "spi_misc_test", + .fops = &spi_test_fops, +}; + +static int vehicle_spi_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct vehicle_spi *vehicle_spi; + int ret = 0; + int id = 0; + + if (!spi) + return -ENOMEM; + + if (g_vehicle_hw == NULL) + return -ENOMEM; + + vehicle_spi = devm_kzalloc(dev, sizeof(*vehicle_spi), GFP_KERNEL); + if (vehicle_spi == NULL) + return -ENOMEM; + + spi_set_drvdata(spi, vehicle_spi); + + vehicle_spi->hw_data = &vehicle_spi_data; + spi->bits_per_word = 8; + vehicle_spi->dev = dev; + vehicle_spi->spi = spi; + g_vehicle_hw->vehicle_spi = vehicle_spi; + + ret = spi_setup(spi); + if (ret < 0) { + dev_err(dev, "ERR: fail to setup spi\n"); + return -1; + } + + if (device_property_read_u32(&spi->dev, "id", &id)) { + dev_warn(&spi->dev, "fail to get id, default set 0\n"); + id = 0; + } + + init_completion(&spi_complete); + spi_hw_init(g_vehicle_hw); + +#ifdef CONFIG_VEHICLE_GPIO_MCU_EXPANDER + gpio_mcu_register(spi); +#endif + + return 0; +} + +static void vehicle_spi_remove(struct spi_device *spi) +{ + struct vehicle_spi *vehicle_spi = spi_get_drvdata(spi); + + destroy_workqueue(vehicle_spi->vehicle_wq); +} + +#ifdef CONFIG_OF +static const struct of_device_id vehicle_spi_id[] = { + { .compatible = "rockchip,vehicle-spi", }, + {}, +}; +#endif + +static struct spi_driver vehicle_spi_device_driver = { + .probe = vehicle_spi_probe, + .remove = vehicle_spi_remove, + .driver = { + .name = "vehicle-spi", + .of_match_table = vehicle_spi_id, + } +}; + +static int vehicle_spi_init(void) +{ + int err; + + misc_register(&spi_test_misc); + err = spi_register_driver(&vehicle_spi_device_driver); + if (err) { + pr_err("Failed to register vehicle spi driver\n"); + misc_deregister(&spi_test_misc); + } + + return err; +} +module_init(vehicle_spi_init); + +static void __exit vehicle_spi_exit(void) +{ + misc_deregister(&spi_test_misc); + spi_unregister_driver(&vehicle_spi_device_driver); +} + +module_exit(vehicle_spi_exit); + +MODULE_LICENSE("GPL"); diff --git a/drivers/misc/vehicle/vehicle_core.c b/drivers/misc/vehicle/vehicle_core.c new file mode 100644 index 000000000000..5a800e0f88c4 --- /dev/null +++ b/drivers/misc/vehicle/vehicle_core.c @@ -0,0 +1,418 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "core.h" + +#include "pb_encode.h" +#include "pb_decode.h" +#include "pb.h" +#include "vehiclehalproto.pb.h" +#include "vehicle_protocol_callback.h" +#include "vehicle_core.h" + +#define PROTOCOL_ID 30 + +struct vehicle *g_vehicle_hw; +EXPORT_SYMBOL(g_vehicle_hw); +static struct sock *nlsk; +static int g_user_pid; + +struct vehicle_core_drvdata { + const struct hw_prop_ops *prop_ops; +}; + +/* param in command PWR_REQ */ +enum vehicle_power_request_param { + AP_POWER_REQUEST_PARAM_SHUTDOWN_IMMEDIATELY = 1, + AP_POWER_REQUEST_PARAM_CAN_SLEEP, + AP_POWER_REQUEST_PARAM_SHUTDOWN_ONLY, + AP_POWER_REQUEST_PARAM_SLEEP_IMMEDIATELY, + AP_POWER_REQUEST_PARAM_HIBERNATE_IMMEDIATELY, + AP_POWER_REQUEST_PARAM_CAN_HIBERNATE, +}; + +/* state in command PWR_REQ */ +enum vehicle_power_request_state { + AP_POWER_REQUEST_STATE_ON = 0, + AP_POWER_REQUEST_STATE_SHUTDOWN_PREPARE, + AP_POWER_REQUEST_STATE_CANCEL_SHUTDOWN, + AP_POWER_REQUEST_STATE_FINISHED, +}; + +static struct vehicle_core_drvdata *vehicle_core; +static struct vehicle_property_set property_encode; +static struct vehicle_property_set property_decode; +static struct vehicle_power_req power_req_encode; + +void vehicle_hw_prop_ops_register(const struct hw_prop_ops *prop_ops) +{ + if (!prop_ops) + return; + + if (vehicle_core) + vehicle_core->prop_ops = prop_ops; +} +EXPORT_SYMBOL_GPL(vehicle_hw_prop_ops_register); + +static int vehicle_send_message_core(u32 prop, u32 area, u32 value) +{ + if (vehicle_core && vehicle_core->prop_ops && + vehicle_core->prop_ops->set_control_commands) + vehicle_core->prop_ops->set_control_commands(prop, area, value); + return 0; +} + +static int send_usrmsg(char *pbuf, uint16_t len) +{ + struct sk_buff *nl_skb; + struct nlmsghdr *nlh; + + int ret; + + if (nlsk == NULL || pbuf == NULL) { + pr_err("Invalid parameters nlsk %p pbuf %p\n", nlsk, pbuf); + return -1; + } + + nl_skb = nlmsg_new(len, GFP_ATOMIC); + if (!nl_skb) { + pr_err("netlink alloc failure\n"); + return -1; + } + + nlh = nlmsg_put(nl_skb, 0, 0, 0, len, 0); + if (nlh == NULL) { + pr_err("nlmsg_put failaure\n"); + nlmsg_free(nl_skb); + return -1; + } + + memcpy(nlmsg_data(nlh), pbuf, len); + ret = netlink_unicast(nlsk, nl_skb, g_user_pid, MSG_DONTWAIT); + + return ret; +} + +void vehicle_set_property(u16 prop, u8 index, u32 value, u32 param) +{ + char *buffer; + + buffer = kmalloc(128, GFP_KERNEL); + pr_debug("%s: prop %d, index %d, value %d\n", __func__, prop, index, value); + property_encode.value = value; + switch (prop) { + case VEHICLE_FAN_SPEED: + property_encode.prop = HVAC_FAN_SPEED; + property_encode.area_id = HVAC_ALL; + if (property_encode.value != FAN_SPEED_0 && + property_encode.value != FAN_SPEED_1 && + property_encode.value != FAN_SPEED_2 && + property_encode.value != FAN_SPEED_3 && + property_encode.value != FAN_SPEED_4 && + property_encode.value != FAN_SPEED_5) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_FAN_DIRECTION: + property_encode.prop = HVAC_FAN_DIRECTION; + property_encode.area_id = HVAC_ALL; + if (property_encode.value != FAN_DIRECTION_0 && + property_encode.value != FAN_DIRECTION_1 && + property_encode.value != FAN_DIRECTION_2 && + property_encode.value != FAN_DIRECTION_3) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_AUTO_ON: + property_encode.prop = HVAC_AUTO_ON; + property_encode.area_id = (u32)index; + if (property_encode.value != AUTO_ON && property_encode.value != AUTO_OFF) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_AC: + property_encode.prop = HVAC_AC_ON; + property_encode.area_id = HVAC_ALL; + if (property_encode.value != AC_ON && property_encode.value != AC_OFF) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_RECIRC_ON: + property_encode.prop = HVAC_RECIRC_ON; + property_encode.area_id = HVAC_ALL; + if (property_encode.value != RECIRC_ON && property_encode.value != RECIRC_OFF) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_DEFROST: + property_encode.prop = HVAC_DEFROSTER; + property_encode.area_id = (u32)index; + if (property_encode.value != DEFROST_ON && property_encode.value != DEFROST_OFF) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_AC_TEMP: + property_encode.prop = HVAC_TEMPERATURE_SET; + property_encode.area_id = (u32)index; + break; + case VEHICLE_HVAC_POWER_ON: + property_encode.prop = HVAC_POWER_ON; + property_encode.area_id = (u32)index; + if (property_encode.value != HVAC_ON && property_encode.value != HVAC_OFF) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_SEAT_TEMPERATURE: + property_encode.prop = HVAC_SEAT_TEMPERATURE; + property_encode.area_id = (u32)index; + if (property_encode.value != SEAT_TEMP_0 && + property_encode.value != SEAT_TEMP_1 && + property_encode.value != SEAT_TEMP_2 && + property_encode.value != SEAT_TEMP_3) { + pr_err("input value is not correct, please type correct one\n"); + goto exit; + } + break; + case VEHICLE_GEAR: + property_encode.prop = GEAR_SELECTION; + if (value == VEHICLE_GEAR_DRIVE) + property_encode.value = VEHICLE_GEAR_DRIVE_CLIENT; + else if (value == VEHICLE_GEAR_REVERSE) + property_encode.value = VEHICLE_GEAR_REVERSE_CLIENT; + else if (value == VEHICLE_GEAR_PARKING) + property_encode.value = VEHICLE_GEAR_PARK_CLIENT; + break; + case VEHICLE_TURN_SIGNAL: + property_encode.prop = TURN_SIGNAL_STATE; + break; + case VEHICLE_POWER_STATE_REQ: + if (value != AP_POWER_REQUEST_STATE_ON && + value != AP_POWER_REQUEST_STATE_SHUTDOWN_PREPARE && + value != AP_POWER_REQUEST_STATE_CANCEL_SHUTDOWN && + value != AP_POWER_REQUEST_STATE_FINISHED) { + pr_err("AP_POWER_STATE_REQ: invalid state\n"); + goto exit; + } + if (param != AP_POWER_REQUEST_PARAM_SHUTDOWN_IMMEDIATELY && + param != AP_POWER_REQUEST_PARAM_CAN_SLEEP && + param != AP_POWER_REQUEST_PARAM_SHUTDOWN_ONLY && + param != AP_POWER_REQUEST_PARAM_SLEEP_IMMEDIATELY && + param != AP_POWER_REQUEST_PARAM_HIBERNATE_IMMEDIATELY && + param != AP_POWER_REQUEST_PARAM_CAN_HIBERNATE) { + pr_err("AP_POWER_STATE_REQ: invalid param\n"); + goto exit; + } + + power_req_encode.prop = AP_POWER_STATE_REQ; + power_req_encode.state = value; + power_req_encode.param = param; + break; + default: + pr_err("property %d is not supported\n", prop); + } + + pb_ostream_t stream; + + stream = pb_ostream_from_buffer(buffer, 128); + + emulator_EmulatorMessage send_message = {}; + + send_message.msg_type = emulator_MsgType_SET_PROPERTY_CMD; + send_message.has_status = true; + send_message.status = emulator_Status_RESULT_OK; + if (prop == VEHICLE_POWER_STATE_REQ) { + send_message.value.funcs.encode = &encode_power_state_callback; + send_message.value.arg = &power_req_encode; + } else { + send_message.value.funcs.encode = &encode_value_callback; + send_message.value.arg = &property_encode; + } + if (!pb_encode(&stream, emulator_EmulatorMessage_fields, &send_message)) + pr_err("vehicle protocol encode fail\n"); + send_usrmsg(buffer, stream.bytes_written); +exit: + kfree(buffer); +} +EXPORT_SYMBOL_GPL(vehicle_set_property); + +static void netlink_rcv_msg(struct sk_buff *skb) +{ + struct nlmsghdr *nlh = NULL; + char *umsg = NULL; + char *buffer; + bool status; + size_t len; + int i; + emulator_EmulatorMessage emulator_message; + + buffer = kmalloc(128, GFP_KERNEL); + if (skb->len >= nlmsg_total_size(0)) { + nlh = nlmsg_hdr(skb); + g_user_pid = nlh->nlmsg_pid; + umsg = NLMSG_DATA(nlh); + len = nlh->nlmsg_len - NLMSG_LENGTH(0); + if (umsg) { + for (i = 0; i < len; i++) + pr_debug("%s raw byte %d %d\n", __func__, i, umsg[i]); + memcpy(buffer, umsg, len); + pb_istream_t stream = pb_istream_from_buffer(buffer, len); + + emulator_message.prop.funcs.decode = &decode_prop_callback; + emulator_message.config.funcs.decode = &decode_config_callback; + emulator_message.value.funcs.decode = &decode_value_callback; + emulator_message.value.arg = &property_decode; + + status = pb_decode(&stream, emulator_EmulatorMessage_fields, + &emulator_message); + if (!status) + pr_err("pb_decode failed\n"); + + vehicle_send_message_core(property_decode.prop, property_decode.area_id, + property_decode.value); + } + } + kfree(buffer); +} + +static void create_netlink_vehicle(void) +{ + struct netlink_kernel_cfg cfg = { + .input = netlink_rcv_msg, + }; + + nlsk = netlink_kernel_create(&init_net, PROTOCOL_ID, &cfg); + if (nlsk == NULL) { + pr_err("netlink_kernel_create error !\n"); + return; + } +} + +static struct vehicle_core_drvdata * +vehicle_get_devtree_pdata(struct device *dev) +{ + struct vehicle_core_drvdata *ddata; + + ddata = devm_kzalloc(dev, + sizeof(*ddata), + GFP_KERNEL); + + if (!ddata) + return ERR_PTR(-ENOMEM); + + return ddata; +} + +static struct vehicle * +vehicle_get_dev_pdata(struct device *dev) +{ + struct vehicle *ddata; + + ddata = devm_kzalloc(dev, + sizeof(*ddata), + GFP_KERNEL); + if (!ddata) + return ERR_PTR(-ENOMEM); + + return ddata; +} + +static int vehicle_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct vehicle_core_drvdata *ddata; + + ddata = vehicle_get_devtree_pdata(dev); + if (IS_ERR(ddata)) + return PTR_ERR(ddata); + + g_vehicle_hw = vehicle_get_dev_pdata(dev); + if (IS_ERR(g_vehicle_hw)) + return PTR_ERR(g_vehicle_hw); + + vehicle_core = ddata; + platform_set_drvdata(pdev, ddata); + platform_set_drvdata(pdev, g_vehicle_hw); + + create_netlink_vehicle(); + return 0; +} + +static int vehicle_remove(struct platform_device *pdev) +{ + if (nlsk) + netlink_kernel_release(nlsk); + return 0; +} + +static struct platform_driver vehicle_device_driver = { + .probe = vehicle_probe, + .remove = vehicle_remove, + .driver = { + .name = "vehicle-core", + } +}; +static struct platform_device *core_pdev; + +static int vehicle_init(void) +{ + int err; + + core_pdev = platform_device_alloc("vehicle-core", -1); + if (!core_pdev) { + pr_err("Failed to allocate core vehicle device\n"); + return -ENODEV; + } + + err = platform_device_add(core_pdev); + if (err != 0) { + pr_err("Failed to register core device: %d\n", err); + platform_device_put(core_pdev); + return err; + } + + err = platform_driver_register(&vehicle_device_driver); + if (err) + pr_err("Failed to register vehicle driver\n"); + + return err; +} + +static void __exit vehicle_exit(void) +{ + platform_driver_unregister(&vehicle_device_driver); + platform_device_unregister(core_pdev); +} + +postcore_initcall(vehicle_init); +module_exit(vehicle_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("NXP Semiconductor"); +MODULE_DESCRIPTION("VEHICLE core driver"); diff --git a/drivers/misc/vehicle/vehicle_core.h b/drivers/misc/vehicle/vehicle_core.h new file mode 100644 index 000000000000..3a586c0ce75a --- /dev/null +++ b/drivers/misc/vehicle/vehicle_core.h @@ -0,0 +1,388 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef VEHICLE_CORE_H +#define VEHICLE_CORE_H + +#include + +// seat temperature indexes +#define SEAT_TEMP_LEFT_INDEX 1 +#define SEAT_TEMP_RIGHT_INDEX 4 +// seat temperature values +#define SEAT_TEMP_0 0 // off +#define SEAT_TEMP_1 1 +#define SEAT_TEMP_2 2 +#define SEAT_TEMP_3 3 + +#define DEFROST_ON 1 +#define DEFROST_OFF 0 + +#define AC_ON 1 +#define AC_OFF 0 + +#define AUTO_ON 1 +#define AUTO_OFF 0 + +#define HVAC_ON 1 +#define HVAC_OFF 0 + +#define RECIRC_ON 1 +#define RECIRC_OFF 0 + +#define FAN_SPEED_0 1 +#define FAN_SPEED_1 2 +#define FAN_SPEED_2 3 +#define FAN_SPEED_3 4 +#define FAN_SPEED_4 5 +#define FAN_SPEED_5 6 + +#define FAN_DIRECTION_0 1 +#define FAN_DIRECTION_1 2 +#define FAN_DIRECTION_2 3 +#define FAN_DIRECTION_3 6 + +struct vehicle_property_set { + u32 prop; + u32 area_id; + u64 value; +}; + +struct vehicle_power_req { + u32 prop; + u32 state; + u32 param; +}; + +struct hw_prop_ops { + /* send the control commands to hw */ + void (*set_control_commands)(u32 prop, u32 area, u32 value); +}; + +/* areaId defines - begin */ +#define VEHICLE_AREA_SEAT_ROW_1_LEFT 1 /* 0x0001 */ +#define VEHICLE_AREA_SEAT_ROW_1_CENTER 2 /* 0x0002 */ +#define VEHICLE_AREA_SEAT_ROW_1_RIGHT 4 /* 0x0004 */ +#define VEHICLE_AREA_SEAT_ROW_2_LEFT 16 /* 0x0010 */ +#define VEHICLE_AREA_SEAT_ROW_2_CENTER 32 /* 0x0020 */ +#define VEHICLE_AREA_SEAT_ROW_2_RIGHT 64 /* 0x0040 */ +#define VEHICLE_AREA_SEAT_ROW_3_LEFT 256 /* 0x0100 */ +#define VEHICLE_AREA_SEAT_ROW_3_CENTER 512 /* 0x0200 */ +#define VEHICLE_AREA_SEAT_ROW_3_RIGHT 1024 /* 0x0400 */ + +#define HVAC_LEFT VEHICLE_AREA_SEAT_ROW_1_LEFT +#define HVAC_RIGHT VEHICLE_AREA_SEAT_ROW_1_RIGHT +//Due to VHAL migration from hidl to aidl, details in commit message +#define HVAC_ALL VEHICLE_AREA_SEAT_ROW_1_LEFT +/* areaId defines - end */ + +#define HVAC_FAN_SPEED 356517120 +#define HVAC_FAN_DIRECTION 356517121 +#define HVAC_AUTO_ON 354419978 +#define HVAC_AC_ON 354419973 +#define HVAC_RECIRC_ON 354419976 +#define HVAC_DEFROSTER 320865540 +#define HVAC_TEMPERATURE_SET 358614275 +#define HVAC_POWER_ON 354419984 + +#define TURN_SIGNAL_STATE 289408008 +#define GEAR_SELECTION 289408000 +#define CURRENT_GEAR 289408001 + +#define AP_POWER_STATE_REQ 289475072 +#define AP_POWER_STATE_REPORT 289475073 + +#define VEHICLE_GEAR_DRIVE_CLIENT 8 +#define VEHICLE_GEAR_PARK_CLIENT 4 +#define VEHICLE_GEAR_REVERSE_CLIENT 2 + +#define WATCHDOG_ALIVE 290459441 +#define HVAC_SEAT_TEMPERATURE 356517131 +#define DISPLAY_BRIGHTNESS 289409539 +#define INFO_MAKE 286261505 +#define INFO_MODEL 286261506 +#define POWER_POLICY_REQ 286265121 +#define POWER_POLICY_GROUP_REQ 286265122 +#define CURRENT_POWER_POLICY 286265123 +#define FUEL_DOOR_OPEN 287310600 +#define EV_CHARGE_PORT_OPEN 287310602 +#define EV_CHARGE_PORT_CONNECTED 287310603 +#define PARKING_BRAKE_ON 287310850 +#define PARKING_BRAKE_AUTO_APPLY 287310851 +#define FUEL_LEVEL_LOW 287310853 +#define NIGHT_MODE 287310855 +#define ABS_ACTIVE 287310858 +#define TRACTION_CONTROL_ACTIVE 287310859 +#define SUPPORT_CUSTOMIZE_VENDOR_PERMISSION 287313669 +#define INFO_MODEL_YEAR 289407235 +#define INFO_FUEL_DOOR_LOCATION 289407240 +#define INFO_EV_PORT_LOCATION 289407241 +#define ENGINE_OIL_LEVEL 289407747 +#define IGNITION_STATE 289408009 +#define HVAC_STEERING_WHEEL_HEAT 289408269 +#define HVAC_TEMPERATURE_DISPLAY_UNITS 289408270 +#define DISTANCE_DISPLAY_UNITS 289408512 +#define TIRE_PRESSURE_DISPLAY_UNITS 289408514 +#define VEHICLE_SPEED_DISPLAY_UNITS 289408517 +#define HEADLIGHTS_STATE 289410560 +#define HIGH_BEAM_LIGHTS_STATE 289410561 +#define FOG_LIGHTS_STATE 289410562 +#define PARKING_BRAKE_AUTO_APPLY 287310851 +#define HAZARD_LIGHTS_STATE 289410563 +#define HEADLIGHTS_SWITCH 289410576 +#define HIGH_BEAM_LIGHTS_SWITCH 289410577 +#define FOG_LIGHTS_SWITCH 289410578 +#define HAZARD_LIGHTS_SWITCH 289410579 +#define CLUSTER_SWITCH_UI 289410868 +#define CLUSTER_REQUEST_DISPLAY 289410871 +#define POWER_POLICY_GROUP_REQ 286265122 +#define CURRENT_POWER_POLICY 286265123 +#define ELECTRONIC_TOLL_COLLECTION_CARD_TYPE 289410873 +#define ELECTRONIC_TOLL_COLLECTION_CARD_STATUS 289410874 +#define INFO_FUEL_TYPE 289472773 +#define INFO_EV_CONNECTOR_TYPE 289472775 +#define INFO_DRIVER_SEAT 356516106 +#define INFO_EXTERIOR_DIMENSIONS 289472779 +#define INFO_MULTI_EV_PORT_LOCATIONS 289472780 +#define HW_KEY_INPUT 289475088 +#define HW_ROTARY_INPUT 289475104 +#define HW_CUSTOM_INPUT 289475120 +#define EVS_SERVICE_REQUEST 289476368 +#define CLUSTER_DISPLAY_STATE 289476405 +#define EPOCH_TIME 290457094 +#define VHAL_HEARTBEAT 290459443 +#define WHEEL_TICK 290521862 +#define INFO_FUEL_CAPACITY 291504388 +#define INFO_EV_BATTERY_CAPACITY 291504390 +#define PERF_ODOMETER 291504644 +#define PERF_VEHICLE_SPEED 291504647 +#define PERF_STEERING_ANGLE 291504649 +#define PERF_REAR_STEERING_ANGLE 291504656 +#define ENGINE_OIL_TEMP 291504900 +#define ENGINE_RPM 291504901 +#define FUEL_LEVEL 291504903 +#define RANGE_REMAINING 291504904 +#define EV_BATTERY_LEVEL 291504905 +#define EV_BATTERY_INSTANTANEOUS_CHARGE_RATE 291504908 +#define ENV_OUTSIDE_TEMPERATURE 291505923 +#define HVAC_TEMPERATURE_VALUE_SUGGESTION 291570965 +#define STORAGE_ENCRYPTION_BINDING_SEED 292554247 +#define CLUSTER_NAVIGATION_STATE 292556600 +#define INITIAL_USER_INFO 299896583 +#define SWITCH_USER 299896584 +#define CREATE_USER 299896585 +#define REMOVE_USER 299896586 +#define USER_IDENTIFICATION_ASSOCIATION 299896587 +#define WATCHDOG_TERMINATED_PROCESS 299896626 +#define CLUSTER_REPORT_STATE 299896630 +#define HVAC_ELECTRIC_DEFROSTER_ON 320865556 +#define WINDOW_LOCK 320867268 +#define VEHICLE_MAP_SERVICE 299895808 +#define WINDOW_POS 322964416 +#define HVAC_MAX_AC_ON 354419974 +#define HVAC_MAX_DEFROST_ON 354419975 +#define HVAC_DUAL_ON 354419977 +#define HVAC_AUTO_RECIRC_ON 354419986 +#define HVAC_SEAT_VENTILATION 356517139 +#define HVAC_ELECTRIC_DEFROSTER_ON 320865556 +#define SEAT_OCCUPANCY 356518832 +#define HVAC_FAN_DIRECTION_AVAILABLE 356582673 +#define DOOR_LOCK 371198722 +#define DOOR_POS 373295872 +#define TIRE_PRESSURE 392168201 +#define CRITICALLY_LOW_TIRE_PRESSURE 392168202 +#define FRONT_FOG_LIGHTS_STATE 289410875 +#define FRONT_FOG_LIGHTS_SWITCH 289410876 +#define REAR_FOG_LIGHTS_STATE 289410877 +#define REAR_FOG_LIGHTS_SWITCH 289410878 +#define EV_CHARGE_CURRENT_DRAW_LIMIT 291508031 +#define EV_CHARGE_PERCENT_LIMIT 291508032 +#define EV_CHARGE_STATE 289410881 +#define EV_CHARGE_SWITCH 287313730 +#define EV_CHARGE_TIME_REMAINING 289410883 +#define EV_REGENERATIVE_BRAKING_STATE 289410884 +#define TRAILER_PRESENT 289410885 +#define VEHICLE_CURB_WEIGHT 289410886 +#define READING_LIGHTS_SWITCH 356519684 +#define SEAT_BELT_HEIGHT_POS 356518787 +#define WINDOW_MOVE 322964417 +#define MIRROR_AUTO_TILT_ENABLED 337644359 +#define SEAT_FORE_AFT_POS 356518789 +#define EV_BATTERY_DISPLAY_UNITS 289408515 +#define SEAT_HEIGHT_MOVE 356518796 +#define INFO_VIN 286261504 +#define SEAT_HEADREST_ANGLE_POS 356518807 +#define SEAT_HEADREST_ANGLE_MOVE 356518808 +#define ENGINE_COOLANT_TEMP 291504897 +#define MIRROR_LOCK 287312708 +#define HVAC_ACTUAL_FAN_SPEED_RPM 356517135 +#define SEAT_BELT_HEIGHT_MOVE 356518788 +#define SEAT_FORE_AFT_MOVE 356518790 +#define SEAT_BACKREST_ANGLE_2_MOVE 356518794 +#define SEAT_HEIGHT_POS 356518795 +#define SEAT_TILT_POS 356518799 +#define SEAT_DEPTH_MOVE 356518798 +#define SEAT_LUMBAR_FORE_AFT_POS 356518801 +#define SEAT_EASY_ACCESS_ENABLED 354421661 +#define SEAT_LUMBAR_FORE_AFT_MOVE 356518802 +#define SEAT_AIRBAG_ENABLED 354421662 +#define AUTOMATIC_EMERGENCY_BRAKING_ENABLED 287313920 +#define SEAT_LUMBAR_SIDE_SUPPORT_POS 356518803 +#define SEAT_LUMBAR_SIDE_SUPPORT_MOVE 356518804 +#define SEAT_MEMORY_SELECT 356518784 +#define HANDS_ON_DETECTION_DRIVER_STATE 289411095 +#define SEAT_HEADREST_HEIGHT_MOVE 356518806 +#define SEAT_HEADREST_FORE_AFT_POS 356518809 +#define MIRROR_FOLD 287312709 +#define SEAT_HEADREST_FORE_AFT_MOVE 356518810 +#define GLOVE_BOX_LOCKED 354421745 +#define SEAT_TILT_MOVE 356518800 +#define MIRROR_Z_POS 339741504 +#define HVAC_SIDE_MIRROR_HEAT 339739916 +#define SEAT_LUMBAR_VERTICAL_POS 356518817 +#define SEAT_FOOTWELL_LIGHTS_SWITCH 356518812 +#define WINDSHIELD_WIPERS_STATE 322964422 +#define SEAT_DEPTH_POS 356518797 +#define SEAT_HEADREST_HEIGHT_POS_V2 356518820 +#define EV_CURRENT_BATTERY_CAPACITY 291504909 +#define EV_BRAKE_REGENERATION_LEVEL 289408012 +#define SEAT_LUMBAR_VERTICAL_MOVE 356518818 +#define SEAT_WALK_IN_POS 356518819 +#define SEAT_BELT_BUCKLED 354421634 +#define SEAT_BACKREST_ANGLE_1_POS 356518791 +#define LANE_KEEP_ASSIST_STATE 289411081 +#define LANE_CENTERING_ASSIST_ENABLED 287313930 +#define SEAT_BACKREST_ANGLE_1_MOVE 356518792 +#define SEAT_BACKREST_ANGLE_2_POS 356518793 +#define MIRROR_AUTO_FOLD_ENABLED 337644358 +#define BLIND_SPOT_WARNING_ENABLED 287313924 +#define EV_STOPPING_MODE 289408013 +#define STEERING_WHEEL_LIGHTS_STATE 289410828 +#define MIRROR_Z_MOVE 339741505 +#define SEAT_CUSHION_SIDE_SUPPORT_POS 356518815 +#define MIRROR_Y_MOVE 339741507 +#define AUTOMATIC_EMERGENCY_BRAKING_STATE 289411073 +#define FORWARD_COLLISION_WARNING_ENABLED 287313922 +#define SEAT_MEMORY_SET 356518785 +#define HANDS_ON_DETECTION_WARNING 289411096 +#define WINDSHIELD_WIPERS_PERIOD 322964421 +#define SEAT_FOOTWELL_LIGHTS_STATE 356518811 +#define DOOR_CHILD_LOCK_ENABLED 371198723 +#define DOOR_MOVE 373295873 +#define WINDSHIELD_WIPERS_SWITCH 322964423 +#define STEERING_WHEEL_DEPTH_POS 289410016 +#define GLOVE_BOX_DOOR_POS 356518896 +#define STEERING_WHEEL_HEIGHT_POS 289410018 +#define STEERING_WHEEL_THEFT_LOCK_ENABLED 287312868 +#define SEAT_CUSHION_SIDE_SUPPORT_MOVE 356518816 +#define CABIN_LIGHTS_STATE 289410817 +#define STEERING_WHEEL_EASY_ACCESS_ENABLED 287312870 +#define CABIN_LIGHTS_SWITCH 289410818 +#define STEERING_WHEEL_HEIGHT_MOVE 289410019 +#define STEERING_WHEEL_LIGHTS_SWITCH 289410829 +#define FUEL_CONSUMPTION_UNITS_DISTANCE_OVER_VOLUME 287311364 +#define MIRROR_Y_POS 339741506 +#define LOCATION_CHARACTERIZATION 289410064 +#define EMERGENCY_LANE_KEEP_ASSIST_STATE 289411086 +#define CRUISE_CONTROL_TYPE 289411088 +#define CRUISE_CONTROL_STATE 289411089 +#define STEERING_WHEEL_LOCKED 287312869 +#define ADAPTIVE_CRUISE_CONTROL_TARGET_TIME_GAP 289411092 +#define ADAPTIVE_CRUISE_CONTROL_LEAD_VEHICLE_MEASURED_DISTANCE 289411093 +#define HANDS_ON_DETECTION_ENABLED 287313942 +#define FORWARD_COLLISION_WARNING_STATE 289411075 +#define STEERING_WHEEL_DEPTH_MOVE 289410017 +#define EMERGENCY_LANE_KEEP_ASSIST_ENABLED 287313933 +#define GENERAL_SAFETY_REGULATION_COMPLIANCE 289410887 +#define LANE_CENTERING_ASSIST_STATE 289411084 +#define CRUISE_CONTROL_ENABLED 287313935 +#define PERF_VEHICLE_SPEED_DISPLAY 291504648 +#define HVAC_TEMPERATURE_CURRENT 358614274 +#define LANE_KEEP_ASSIST_ENABLED 287313928 +#define READING_LIGHTS_STATE 356519683 +#define CRUISE_CONTROL_TARGET_SPEED 291508243 +#define ENGINE_IDLE_AUTO_STOP_ENABLED 287310624 +#define BLIND_SPOT_WARNING_STATE 339742725 +#define LANE_DEPARTURE_WARNING_ENABLED 287313926 +#define FUEL_VOLUME_DISPLAY_UNITS 289408513 +#define LANE_DEPARTURE_WARNING_STATE 289411079 + +/* (int)(0x104 | VehiclePropertyGroup:: + * VENDOR | VehiclePropertyType::STRING | VehicleArea::GLOBAL); + */ +#define VENDOR_EXTENSION_STRING_PROPERTY 554696964 +/* (int)(0x101 | VehiclePropertyGroup:: + * VENDOR | VehiclePropertyType::BOOLEAN | VehicleArea::DOOR); + */ +#define VENDOR_EXTENSION_BOOLEAN_PROPERTY 639631617 +/* (int)(0x102 | VehiclePropertyGroup:: + * VENDOR | VehiclePropertyType::FLOAT | VehicleArea::SEAT); + */ +#define VENDOR_EXTENSION_FLOAT_PROPERTY 627048706 +/* (int)(0x103 | VehiclePropertyGroup:: + * VENDOR | VehiclePropertyType::INT32 | VehicleArea::WINDOW); + */ +#define VENDOR_EXTENSION_INT_PROPERTY 591397123 +/* 0x1111 | VehiclePropertyGroup:: + * VENDOR | VehicleArea::GLOBAL | VehiclePropertyType::MIXED; + */ +#define kMixedTypePropertyForTest 568332561 + +/*vehicle_event_type: stateType in command VSTATE*/ +enum vehicle_event_type { + VEHICLE_AC = 0, + VEHICLE_AUTO_ON, + VEHICLE_AC_TEMP, + VEHICLE_FAN_SPEED, + VEHICLE_FAN_DIRECTION, + VEHICLE_RECIRC_ON, + VEHICLE_HEATER, + VEHICLE_DEFROST, + VEHICLE_HVAC_POWER_ON, + VEHICLE_MUTE, + VEHICLE_VOLUME, + VEHICLE_DOOR, + VEHICLE_RVC, + VEHICLE_LIGHT, + VEHICLE_GEAR, + VEHICLE_TURN_SIGNAL, + VEHICLE_POWER_STATE_REQ, + VEHICLE_SEAT_TEMPERATURE, + VEHICLE_UNSUPPORTED = -1, +}; + +/*vehicle_event_gear: stateValue of type VEHICLE_GEAR*/ +enum vehicle_event_gear { + VEHICLE_GEAR_NONE = 0, + VEHICLE_GEAR_PARKING, + VEHICLE_GEAR_REVERSE, + VEHICLE_GEAR_NEUTRAL, + VEHICLE_GEAR_DRIVE, + VEHICLE_GEAR_FIRST, + VEHICLE_GEAR_SECOND, + VEHICLE_GEAR_SPORT, + VEHICLE_GEAR_MANUAL_1, + VEHICLE_GEAR_MANUAL_2, + VEHICLE_GEAR_MANUAL_3, + VEHICLE_GEAR_MANUAL_4, + VEHICLE_GEAR_MANUAL_5, + VEHICLE_GEAR_MANUAL_6, +}; + +extern struct net init_net; +extern void vehicle_hw_prop_ops_register(const struct hw_prop_ops *prop_ops); +extern void vehicle_set_property(u16 prop, u8 index, u32 value, u32 param); + + +#endif diff --git a/drivers/misc/vehicle/vehicle_dummy_hw.c b/drivers/misc/vehicle/vehicle_dummy_hw.c new file mode 100644 index 000000000000..90d43bb30766 --- /dev/null +++ b/drivers/misc/vehicle/vehicle_dummy_hw.c @@ -0,0 +1,916 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include "vehicle_core.h" +#include "core.h" + +/* Miscellaneous external connector */ +#define EXTCON_VEHICLE_RPMSG_REGISTER 58 +#define EXTCON_VEHICLE_RPMSG_EVENT 59 +//VEHICLE is in state parking +#define GEAR_0 1 +//VEHICLE is in state reverse +#define GEAR_1 2 +//VEHICLE is in state neutral +#define GEAR_3 3 +//VEHICLE is in state driver +#define GEAR_2 4 +// no turn signal +#define TURN_0 0 +// left turn signal +#define TURN_1 1 +// right turn signal +#define TURN_2 2 + +#define POWER_REQ_STATE_ON 0 +#define POWER_REQ_STATE_SHUTDOWN_PREPARE 1 +#define POWER_REQ_STATE_CANCEL_SHUTDOWN 2 +#define POWER_REQ_STATE_FINISHED 3 + +#define POWER_REQ_PARAM_SHUTDOWN_IMMEDIATELY 1 +#define POWER_REQ_PARAM_CAN_SLEEP 2 +#define POWER_REQ_PARAM_SHUTDOWN_ONLY 3 +#define POWER_REQ_PARAM_SLEEP_IMMEDIATELY 4 +#define POWER_REQ_PARAM_HIBERNATE_IMMEDIATELY 5 +#define POWER_REQ_PARAM_CAN_HIBERNATE 6 + +// temperature set from hardware on Android OREO and PIE uses below indexes +#define AC_TEMP_LEFT_INDEX 1 +#define AC_TEMP_RIGHT_INDEX 4 + +// temperature set from APP on Android PIE uses below indexes +#define PIE_AC_TEMP_LEFT_INDEX 49 +#define PIE_AC_TEMP_RIGHT_INDEX 68 + +#ifdef CONFIG_EXTCON +static const unsigned int rk_vehicle_dummy_extcon_register_cables[] = { + EXTCON_VEHICLE_RPMSG_REGISTER, + EXTCON_NONE, +}; +static const unsigned int rk_vehicle_dummy_extcon_event_cables[] = { + EXTCON_VEHICLE_RPMSG_EVENT, + EXTCON_NONE, +}; +static struct extcon_dev *rg_edev; +static struct extcon_dev *ev_edev; +#endif + +static struct class *vehicle_dummy_class; + +static void mcu_set_control_commands(u32 prop, u32 area, u32 value) +{ + pr_debug("%s: prop %d, area %d, value %d\n", __func__, prop, area, value); + switch (prop) { + case HVAC_FAN_SPEED: + pr_info("set fan speed with value %d\n", value); + g_vehicle_hw->vehicle_data.fan_speed = value; + break; + case HVAC_FAN_DIRECTION: + pr_info("set fan direction with value %d\n", value); + g_vehicle_hw->vehicle_data.fan_direction = value; + break; + case HVAC_AUTO_ON: + pr_info("set fan auto on with value %d\n", value); + g_vehicle_hw->vehicle_data.auto_on = value; + break; + case HVAC_AC_ON: + pr_info("set fan ac on with value %d\n", value); + g_vehicle_hw->vehicle_data.ac_on = value; + break; + case HVAC_RECIRC_ON: + pr_info("set fan recirc on with value %d\n", value); + g_vehicle_hw->vehicle_data.recirc_on = value; + break; + case HVAC_DEFROSTER: + pr_info("set defroster index %d with value %d\n", area, value); + if (area == 1) + g_vehicle_hw->vehicle_data.defrost_left = value; + else + g_vehicle_hw->vehicle_data.defrost_right = value; + break; + case HVAC_TEMPERATURE_SET: + pr_info("set temp index %d with value %d\n", area, value); + if (area == AC_TEMP_LEFT_INDEX) + g_vehicle_hw->vehicle_data.temp_left = value; + else if (area == AC_TEMP_RIGHT_INDEX) + g_vehicle_hw->vehicle_data.temp_right = value; + break; + case HVAC_POWER_ON: + pr_info("set hvac power on with value %d\n", value); + g_vehicle_hw->vehicle_data.hvac_on = value; + break; + case AP_POWER_STATE_REPORT: + pr_info("receive power state report with value %d\n", value); + break; + case AP_POWER_STATE_REQ: + // Proper action is TBD + break; + case WATCHDOG_ALIVE: + case DISPLAY_BRIGHTNESS: + // Proper action is TBD + break; + case HVAC_SEAT_TEMPERATURE: + pr_info("set seat temperature index %d with value %d\n", area, value); + if (area == SEAT_TEMP_LEFT_INDEX) + g_vehicle_hw->vehicle_data.seat_temp_left = value; + else if (area == SEAT_TEMP_RIGHT_INDEX) + g_vehicle_hw->vehicle_data.seat_temp_right = value; + else + pr_err("unknown index: %d:%d:%d!\n", prop, area, value); + break; + case GEAR_SELECTION: + case TURN_SIGNAL_STATE: + case CURRENT_GEAR: + // GEAR is handled in mxc vehicle driver + break; + case INFO_MAKE: + case INFO_MODEL: + case POWER_POLICY_REQ: + case FUEL_DOOR_OPEN: + case EV_CHARGE_PORT_OPEN: + case EV_CHARGE_PORT_CONNECTED: + case PARKING_BRAKE_ON: + case FUEL_LEVEL_LOW: + case NIGHT_MODE: + case ABS_ACTIVE: + case TRACTION_CONTROL_ACTIVE: + case SUPPORT_CUSTOMIZE_VENDOR_PERMISSION: + case INFO_MODEL_YEAR: + case INFO_FUEL_DOOR_LOCATION: + case INFO_EV_PORT_LOCATION: + case ENGINE_OIL_LEVEL: + case IGNITION_STATE: + case HVAC_STEERING_WHEEL_HEAT: + case HVAC_TEMPERATURE_DISPLAY_UNITS: + case DISTANCE_DISPLAY_UNITS: + case TIRE_PRESSURE_DISPLAY_UNITS: + case VEHICLE_SPEED_DISPLAY_UNITS: + case HEADLIGHTS_STATE: + case HIGH_BEAM_LIGHTS_STATE: + case FOG_LIGHTS_STATE: + case PARKING_BRAKE_AUTO_APPLY: + case HAZARD_LIGHTS_STATE: + case HEADLIGHTS_SWITCH: + case HIGH_BEAM_LIGHTS_SWITCH: + case FOG_LIGHTS_SWITCH: + case HAZARD_LIGHTS_SWITCH: + case CLUSTER_SWITCH_UI: + case CLUSTER_REQUEST_DISPLAY: + case POWER_POLICY_GROUP_REQ: + case CURRENT_POWER_POLICY: + case ELECTRONIC_TOLL_COLLECTION_CARD_TYPE: + case ELECTRONIC_TOLL_COLLECTION_CARD_STATUS: + case INFO_FUEL_TYPE: + case INFO_EV_CONNECTOR_TYPE: + case INFO_DRIVER_SEAT: + case INFO_EXTERIOR_DIMENSIONS: + case INFO_MULTI_EV_PORT_LOCATIONS: + case HW_KEY_INPUT: + case HW_ROTARY_INPUT: + case HW_CUSTOM_INPUT: + case EVS_SERVICE_REQUEST: + case CLUSTER_DISPLAY_STATE: + case EPOCH_TIME: + case VHAL_HEARTBEAT: + case WHEEL_TICK: + case INFO_FUEL_CAPACITY: + case INFO_EV_BATTERY_CAPACITY: + case PERF_ODOMETER: + case PERF_VEHICLE_SPEED: + case PERF_STEERING_ANGLE: + case PERF_REAR_STEERING_ANGLE: + case ENGINE_OIL_TEMP: + case ENGINE_RPM: + case FUEL_LEVEL: + case RANGE_REMAINING: + case EV_BATTERY_LEVEL: + case EV_BATTERY_INSTANTANEOUS_CHARGE_RATE: + case ENV_OUTSIDE_TEMPERATURE: + case HVAC_TEMPERATURE_VALUE_SUGGESTION: + case STORAGE_ENCRYPTION_BINDING_SEED: + case CLUSTER_NAVIGATION_STATE: + case INITIAL_USER_INFO: + case SWITCH_USER: + case CREATE_USER: + case REMOVE_USER: + case USER_IDENTIFICATION_ASSOCIATION: + case WATCHDOG_TERMINATED_PROCESS: + case CLUSTER_REPORT_STATE: + case WINDOW_LOCK: + case VEHICLE_MAP_SERVICE: + case WINDOW_POS: + case HVAC_MAX_AC_ON: + case HVAC_MAX_DEFROST_ON: + case HVAC_DUAL_ON: + case HVAC_AUTO_RECIRC_ON: + case HVAC_SEAT_VENTILATION: + case HVAC_ELECTRIC_DEFROSTER_ON: + case SEAT_OCCUPANCY: + case HVAC_FAN_DIRECTION_AVAILABLE: + case DOOR_LOCK: + case DOOR_POS: + case TIRE_PRESSURE: + case CRITICALLY_LOW_TIRE_PRESSURE: + case FRONT_FOG_LIGHTS_STATE: + case FRONT_FOG_LIGHTS_SWITCH: + case REAR_FOG_LIGHTS_STATE: + case REAR_FOG_LIGHTS_SWITCH: + case EV_CHARGE_CURRENT_DRAW_LIMIT: + case EV_CHARGE_PERCENT_LIMIT: + case EV_CHARGE_STATE: + case EV_CHARGE_SWITCH: + case EV_CHARGE_TIME_REMAINING: + case EV_REGENERATIVE_BRAKING_STATE: + case TRAILER_PRESENT: + case VEHICLE_CURB_WEIGHT: + // Proper action is TBD + break; + case VENDOR_EXTENSION_STRING_PROPERTY: + case VENDOR_EXTENSION_BOOLEAN_PROPERTY: + case VENDOR_EXTENSION_FLOAT_PROPERTY: + case VENDOR_EXTENSION_INT_PROPERTY: + case READING_LIGHTS_SWITCH: + case SEAT_BELT_HEIGHT_POS: + case WINDOW_MOVE: + case MIRROR_AUTO_TILT_ENABLED: + case SEAT_FORE_AFT_POS: + case EV_BATTERY_DISPLAY_UNITS: + case SEAT_HEIGHT_MOVE: + case INFO_VIN: + case SEAT_HEADREST_ANGLE_POS: + case SEAT_HEADREST_ANGLE_MOVE: + case ENGINE_COOLANT_TEMP: + case MIRROR_LOCK: + case HVAC_ACTUAL_FAN_SPEED_RPM: + case SEAT_BELT_HEIGHT_MOVE: + case SEAT_FORE_AFT_MOVE: + case SEAT_BACKREST_ANGLE_2_MOVE: + case SEAT_HEIGHT_POS: + case SEAT_TILT_POS: + case SEAT_DEPTH_MOVE: + case SEAT_LUMBAR_FORE_AFT_POS: + case SEAT_EASY_ACCESS_ENABLED: + case SEAT_LUMBAR_FORE_AFT_MOVE: + case SEAT_AIRBAG_ENABLED: + case AUTOMATIC_EMERGENCY_BRAKING_ENABLED: + case SEAT_LUMBAR_SIDE_SUPPORT_POS: + case SEAT_LUMBAR_SIDE_SUPPORT_MOVE: + case SEAT_MEMORY_SELECT: + case HANDS_ON_DETECTION_DRIVER_STATE: + case SEAT_HEADREST_HEIGHT_MOVE: + case SEAT_HEADREST_FORE_AFT_POS: + case MIRROR_FOLD: + case SEAT_HEADREST_FORE_AFT_MOVE: + case GLOVE_BOX_LOCKED: + case SEAT_TILT_MOVE: + case MIRROR_Z_POS: + case HVAC_SIDE_MIRROR_HEAT: + case SEAT_LUMBAR_VERTICAL_POS: + case SEAT_FOOTWELL_LIGHTS_SWITCH: + case WINDSHIELD_WIPERS_STATE: + case SEAT_DEPTH_POS: + case SEAT_HEADREST_HEIGHT_POS_V2: + case EV_CURRENT_BATTERY_CAPACITY: + case EV_BRAKE_REGENERATION_LEVEL: + case SEAT_LUMBAR_VERTICAL_MOVE: + case SEAT_WALK_IN_POS: + case SEAT_BELT_BUCKLED: + case SEAT_BACKREST_ANGLE_1_POS: + case LANE_KEEP_ASSIST_STATE: + case LANE_CENTERING_ASSIST_ENABLED: + case SEAT_BACKREST_ANGLE_1_MOVE: + case SEAT_BACKREST_ANGLE_2_POS: + case MIRROR_AUTO_FOLD_ENABLED: + case BLIND_SPOT_WARNING_ENABLED: + case EV_STOPPING_MODE: + case STEERING_WHEEL_LIGHTS_STATE: + case MIRROR_Z_MOVE: + case SEAT_CUSHION_SIDE_SUPPORT_POS: + case MIRROR_Y_MOVE: + case AUTOMATIC_EMERGENCY_BRAKING_STATE: + case FORWARD_COLLISION_WARNING_ENABLED: + case SEAT_MEMORY_SET: + case HANDS_ON_DETECTION_WARNING: + case WINDSHIELD_WIPERS_PERIOD: + case SEAT_FOOTWELL_LIGHTS_STATE: + case DOOR_CHILD_LOCK_ENABLED: + case DOOR_MOVE: + case WINDSHIELD_WIPERS_SWITCH: + case STEERING_WHEEL_DEPTH_POS: + case GLOVE_BOX_DOOR_POS: + case STEERING_WHEEL_HEIGHT_POS: + case STEERING_WHEEL_THEFT_LOCK_ENABLED: + case SEAT_CUSHION_SIDE_SUPPORT_MOVE: + case CABIN_LIGHTS_STATE: + case STEERING_WHEEL_EASY_ACCESS_ENABLED: + case CABIN_LIGHTS_SWITCH: + case STEERING_WHEEL_HEIGHT_MOVE: + case STEERING_WHEEL_LIGHTS_SWITCH: + case FUEL_CONSUMPTION_UNITS_DISTANCE_OVER_VOLUME: + case MIRROR_Y_POS: + case LOCATION_CHARACTERIZATION: + case EMERGENCY_LANE_KEEP_ASSIST_STATE: + case CRUISE_CONTROL_TYPE: + case CRUISE_CONTROL_STATE: + case STEERING_WHEEL_LOCKED: + case ADAPTIVE_CRUISE_CONTROL_TARGET_TIME_GAP: + case ADAPTIVE_CRUISE_CONTROL_LEAD_VEHICLE_MEASURED_DISTANCE: + case HANDS_ON_DETECTION_ENABLED: + case FORWARD_COLLISION_WARNING_STATE: + case STEERING_WHEEL_DEPTH_MOVE: + case EMERGENCY_LANE_KEEP_ASSIST_ENABLED: + case GENERAL_SAFETY_REGULATION_COMPLIANCE: + case LANE_CENTERING_ASSIST_STATE: + case CRUISE_CONTROL_ENABLED: + case PERF_VEHICLE_SPEED_DISPLAY: + case HVAC_TEMPERATURE_CURRENT: + case LANE_KEEP_ASSIST_ENABLED: + case READING_LIGHTS_STATE: + case CRUISE_CONTROL_TARGET_SPEED: + case ENGINE_IDLE_AUTO_STOP_ENABLED: + case BLIND_SPOT_WARNING_STATE: + case LANE_DEPARTURE_WARNING_ENABLED: + case FUEL_VOLUME_DISPLAY_UNITS: + case LANE_DEPARTURE_WARNING_STATE: + case kMixedTypePropertyForTest: + // Proper action is TBD + break; + default: + pr_err("this type is not correct: %d:%d:%d!\n", prop, area, value); + } +} + +static ssize_t turn_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.turn); +} + +/* echo 0/1/2(none/left/right) > /sys/devices/platform/vehicle-dummy/turn*/ +static ssize_t turn_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 turn; + + if (!size) + return -EINVAL; + turn = strtoul(buf, NULL, 10); + if (turn != TURN_0 && turn != TURN_1 && turn != TURN_2) { + pr_err("input value is not correct, please type correct one\n"); + return -EINVAL; + } + if (turn != g_vehicle_hw->vehicle_data.turn) { + g_vehicle_hw->vehicle_data.turn = turn; + vehicle_set_property(VEHICLE_TURN_SIGNAL, 0, turn, 0); + } + return size; +} +static DEVICE_ATTR_RW(turn); + +static ssize_t gear_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.gear); +} + +/*echo 1/2/4(parking/reverse/drive) > /sys/devices/platform/vehicle-dummy/gear*/ +static ssize_t gear_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 gear; + + if (!size) + return -EINVAL; + gear = strtoul(buf, NULL, 10); + if (gear != g_vehicle_hw->vehicle_data.gear) { + g_vehicle_hw->vehicle_data.gear = gear; + vehicle_set_property(VEHICLE_GEAR, 0, gear, 0); + if (gear != GEAR_0 && gear != GEAR_1 && gear != GEAR_2) { + pr_err("input value is not correct, please type correct one\n"); + return -EINVAL; + } +#ifdef CONFIG_EXTCON + if (gear == VEHICLE_GEAR_DRIVE) + extcon_set_state_sync(ev_edev, EXTCON_VEHICLE_RPMSG_EVENT, 0); + else if (gear == VEHICLE_GEAR_REVERSE) + extcon_set_state_sync(ev_edev, EXTCON_VEHICLE_RPMSG_EVENT, 1); +#endif + } + return size; +} + +static DEVICE_ATTR_RW(gear); + +static ssize_t temp_left_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", g_vehicle_hw->vehicle_data.temp_left); +} + +/*echo 1100713529 > /sys/devices/platform/vehicle-dummy/temp_left*/ +static ssize_t temp_left_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 temp; + + if (!size) + return -EINVAL; + temp = strtoul(buf, NULL, 10); + if (temp != g_vehicle_hw->vehicle_data.temp_left) { + g_vehicle_hw->vehicle_data.temp_left = temp; + vehicle_set_property(VEHICLE_AC_TEMP, AC_TEMP_LEFT_INDEX, temp, 0); + } + return size; +} + +static DEVICE_ATTR_RW(temp_left); + +static ssize_t temp_right_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", g_vehicle_hw->vehicle_data.temp_right); +} + +/*echo 1100713529 > /sys/devices/platform/vehicle-dummy/temp_right*/ +static ssize_t temp_right_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 temp; + + if (!size) + return -EINVAL; + temp = strtoul(buf, NULL, 10); + if (temp != g_vehicle_hw->vehicle_data.temp_right) { + g_vehicle_hw->vehicle_data.temp_right = temp; + vehicle_set_property(VEHICLE_AC_TEMP, AC_TEMP_RIGHT_INDEX, temp, 0); + } + return size; +} + +static DEVICE_ATTR_RW(temp_right); + +static ssize_t seat_temp_left_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", g_vehicle_hw->vehicle_data.seat_temp_left); +} + +/*echo 0/1/2/3 > /sys/devices/platform/vehicle-dummy/seat_temp_left*/ +static ssize_t seat_temp_left_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 seat_temp; + + if (!size) + return -EINVAL; + seat_temp = strtoul(buf, NULL, 10); + + if (seat_temp != g_vehicle_hw->vehicle_data.seat_temp_left) { + g_vehicle_hw->vehicle_data.seat_temp_left = seat_temp; + vehicle_set_property(VEHICLE_SEAT_TEMPERATURE, SEAT_TEMP_LEFT_INDEX, seat_temp, 0); + } + return size; +} + +static DEVICE_ATTR_RW(seat_temp_left); + +static ssize_t seat_temp_right_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", g_vehicle_hw->vehicle_data.seat_temp_right); +} + +/*echo 0/1/2/3 > /sys/devices/platform/vehicle-dummy/seat_temp_right*/ +static ssize_t seat_temp_right_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 seat_temp; + + if (!size) + return -EINVAL; + seat_temp = strtoul(buf, NULL, 10); + if (seat_temp != SEAT_TEMP_0 && seat_temp != SEAT_TEMP_1 && + seat_temp != SEAT_TEMP_2 && seat_temp != SEAT_TEMP_3) { + pr_err("input value is not correct, please type correct one\n"); + return -EINVAL; + } + if (seat_temp != g_vehicle_hw->vehicle_data.seat_temp_right) { + g_vehicle_hw->vehicle_data.seat_temp_right = seat_temp; + vehicle_set_property(VEHICLE_SEAT_TEMPERATURE, SEAT_TEMP_RIGHT_INDEX, seat_temp, 0); + } + return size; +} + +static DEVICE_ATTR_RW(seat_temp_right); + +static ssize_t fan_direction_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.fan_direction); +} + +/*echo 1/2/3/6 > /sys/devices/platform/vehicle-dummy/fan_direction*/ +static ssize_t fan_direction_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 fan_direction; + + if (!size) + return -EINVAL; + fan_direction = strtoul(buf, NULL, 10); + + if (fan_direction != g_vehicle_hw->vehicle_data.fan_direction) { + g_vehicle_hw->vehicle_data.fan_direction = fan_direction; + vehicle_set_property(VEHICLE_FAN_DIRECTION, 0, fan_direction, 0); + } + return size; +} + +static DEVICE_ATTR_RW(fan_direction); + +static ssize_t fan_speed_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.fan_speed); +} + +/*echo 1/2/3/4/5/6 > /sys/devices/platform/vehicle-dummy/fan_speed*/ +static ssize_t fan_speed_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 fan_speed; + + if (!size) + return -EINVAL; + fan_speed = strtoul(buf, NULL, 10); + + if (fan_speed != g_vehicle_hw->vehicle_data.fan_speed) { + g_vehicle_hw->vehicle_data.fan_speed = fan_speed; + vehicle_set_property(VEHICLE_FAN_SPEED, 0, fan_speed, 0); + } + return size; +} + +static DEVICE_ATTR_RW(fan_speed); + +static ssize_t defrost_left_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.defrost_left); +} + +/*echo 0/1 > /sys/devices/platform/vehicle-dummy/defrost_left*/ +static ssize_t defrost_left_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 defrost; + + if (!size) + return -EINVAL; + + defrost = strtoul(buf, NULL, 10); + + if (defrost != g_vehicle_hw->vehicle_data.defrost_left) { + g_vehicle_hw->vehicle_data.defrost_left = defrost; + vehicle_set_property(VEHICLE_DEFROST, 1, defrost, 0); + } + return size; +} + +static DEVICE_ATTR_RW(defrost_left); + +static ssize_t defrost_right_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.defrost_right); +} + +/*echo 0/1 > /sys/devices/platform/vehicle-dummy/defrost_right*/ +static ssize_t defrost_right_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 defrost; + + if (!size) + return -EINVAL; + + defrost = strtoul(buf, NULL, 10); + if (defrost != DEFROST_ON && defrost != DEFROST_OFF) { + pr_err("input value is not correct, please type correct one\n"); + return -EINVAL; + } + if (defrost != g_vehicle_hw->vehicle_data.defrost_right) { + g_vehicle_hw->vehicle_data.defrost_right = defrost; + vehicle_set_property(VEHICLE_DEFROST, 2, defrost, 0); + } + return size; +} + +static DEVICE_ATTR_RW(defrost_right); + +static ssize_t ac_on_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.ac_on); +} + +/*echo 0/1 > /sys/devices/platform/vehicle-dummy/ac_on*/ +static ssize_t ac_on_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 ac_on; + + if (!size) + return -EINVAL; + + ac_on = strtoul(buf, NULL, 10); + + if (ac_on != g_vehicle_hw->vehicle_data.ac_on) { + g_vehicle_hw->vehicle_data.ac_on = ac_on; + vehicle_set_property(VEHICLE_AC, 0, ac_on, 0); + } + return size; +} + +static DEVICE_ATTR_RW(ac_on); + +static ssize_t auto_on_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.auto_on); +} + +/*echo 0/1 > /sys/devices/platform/vehicle-dummy/auto_on*/ +static ssize_t auto_on_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 auto_on; + + if (!size) + return -EINVAL; + + auto_on = strtoul(buf, NULL, 10); + + if (auto_on != g_vehicle_hw->vehicle_data.auto_on) { + g_vehicle_hw->vehicle_data.auto_on = auto_on; + vehicle_set_property(VEHICLE_AUTO_ON, VEHICLE_AREA_SEAT_ROW_1_LEFT, + auto_on, 0); + vehicle_set_property(VEHICLE_AUTO_ON, VEHICLE_AREA_SEAT_ROW_1_RIGHT, + auto_on, 0); + vehicle_set_property(VEHICLE_AUTO_ON, VEHICLE_AREA_SEAT_ROW_2_LEFT, + auto_on, 0); + vehicle_set_property(VEHICLE_AUTO_ON, VEHICLE_AREA_SEAT_ROW_2_RIGHT, + auto_on, 0); + vehicle_set_property(VEHICLE_AUTO_ON, VEHICLE_AREA_SEAT_ROW_2_CENTER, + auto_on, 0); + } + return size; +} + +static DEVICE_ATTR_RW(auto_on); + +static ssize_t hvac_on_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.hvac_on); +} + +/*echo 0/1 > /sys/devices/platform/vehicle-dummy/hvac_on*/ +static ssize_t hvac_on_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 hvac_on; + + if (!size) + return -EINVAL; + + hvac_on = strtoul(buf, NULL, 10); + + if (hvac_on != g_vehicle_hw->vehicle_data.hvac_on) { + g_vehicle_hw->vehicle_data.hvac_on = hvac_on; + vehicle_set_property(VEHICLE_HVAC_POWER_ON, VEHICLE_AREA_SEAT_ROW_1_LEFT, + hvac_on, 0); + vehicle_set_property(VEHICLE_HVAC_POWER_ON, VEHICLE_AREA_SEAT_ROW_1_RIGHT, + hvac_on, 0); + } + return size; +} + +static DEVICE_ATTR_RW(hvac_on); + +static ssize_t recirc_on_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", g_vehicle_hw->vehicle_data.recirc_on); +} + +/* echo 0/1 > /sys/devices/platform/vehicle-dummy/recirc_on*/ +static ssize_t recirc_on_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + u32 recirc_on; + + if (!size) + return -EINVAL; + + recirc_on = strtoul(buf, NULL, 10); + + if (recirc_on != g_vehicle_hw->vehicle_data.recirc_on) { + g_vehicle_hw->vehicle_data.recirc_on = recirc_on; + vehicle_set_property(VEHICLE_RECIRC_ON, 0, recirc_on, 0); + } + return size; +} + +static DEVICE_ATTR_RW(recirc_on); + +static ssize_t power_req_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u %u\n", g_vehicle_hw->vehicle_data.power_req_state, + g_vehicle_hw->vehicle_data.power_req_param); +} + +/* echo "1 1" > /sys/devices/platform/vehicle-dummy/power_req */ +static ssize_t power_req_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + char *ret; + u32 state; + u32 param; + + if (size < 4) { + pr_err("input command format is not correct, please type command like this: \"1 1\"\n"); + return -EINVAL; + } + + state = strtoul(buf, NULL, 10); + ret = strrchr(buf, ' '); + param = strtoul(ret + 1, NULL, 10); + + if (state != POWER_REQ_STATE_ON && + state != POWER_REQ_STATE_SHUTDOWN_PREPARE && + state != POWER_REQ_STATE_CANCEL_SHUTDOWN && + state != POWER_REQ_STATE_FINISHED) { + pr_err("input power request state is not correct, please type correct one\n"); + return -EINVAL; + } + + if (param != POWER_REQ_PARAM_SHUTDOWN_IMMEDIATELY && + param != POWER_REQ_PARAM_CAN_SLEEP && + param != POWER_REQ_PARAM_SHUTDOWN_ONLY && + param != POWER_REQ_PARAM_SLEEP_IMMEDIATELY && + param != POWER_REQ_PARAM_HIBERNATE_IMMEDIATELY && + param != POWER_REQ_PARAM_CAN_HIBERNATE) { + pr_err("input power request param is not correct, please type correct one\n"); + return -EINVAL; + } + + g_vehicle_hw->vehicle_data.power_req_state = state; + g_vehicle_hw->vehicle_data.power_req_param = param; + vehicle_set_property(VEHICLE_POWER_STATE_REQ, 0, state, param); + pr_info("power control with state: %d, param: %d\n", state, param); + return size; +} + +static DEVICE_ATTR_RW(power_req); + +static const struct hw_prop_ops hw_prop_mcu_ops = { + .set_control_commands = mcu_set_control_commands, +}; + +static int vehicle_dummy_hw_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct vehicle_dummy *vehicle_dummy; + int err; + + if (IS_ERR(g_vehicle_hw)) + return PTR_ERR(g_vehicle_hw); + + vehicle_dummy = devm_kzalloc(dev, sizeof(*vehicle_dummy), GFP_KERNEL); + if (vehicle_dummy == NULL) + return -ENOMEM; + + platform_set_drvdata(pdev, vehicle_dummy); + + vehicle_dummy_class = class_create(THIS_MODULE, "vehicle_dummy_hw"); + if (IS_ERR(vehicle_dummy_class)) { + dev_err(dev, "failed to create class.\n"); + return PTR_ERR(vehicle_dummy_class); + } + + err = device_create_file(dev, &dev_attr_recirc_on) || + device_create_file(dev, &dev_attr_hvac_on) || + device_create_file(dev, &dev_attr_auto_on) || + device_create_file(dev, &dev_attr_ac_on) || + device_create_file(dev, &dev_attr_defrost_right) || + device_create_file(dev, &dev_attr_defrost_left) || + device_create_file(dev, &dev_attr_fan_speed) || + device_create_file(dev, &dev_attr_fan_direction) || + device_create_file(dev, &dev_attr_temp_left) || + device_create_file(dev, &dev_attr_temp_right) || + device_create_file(dev, &dev_attr_gear) || + device_create_file(dev, &dev_attr_power_req) || + device_create_file(dev, &dev_attr_seat_temp_left) || + device_create_file(dev, &dev_attr_seat_temp_right) || + device_create_file(dev, &dev_attr_turn); + if (err) + return err; + + vehicle_hw_prop_ops_register(&hw_prop_mcu_ops); + +#ifdef CONFIG_EXTCON + rg_edev = devm_extcon_dev_allocate(dev, rk_vehicle_dummy_extcon_register_cables); + if (IS_ERR(rg_edev)) + dev_err(dev, "failed to allocate extcon device\n"); + + err = devm_extcon_dev_register(dev, rg_edev); + if (err < 0) + dev_err(dev, "failed to register extcon device\n"); + + ev_edev = devm_extcon_dev_allocate(dev, rk_vehicle_dummy_extcon_event_cables); + if (IS_ERR(ev_edev)) + dev_err(dev, "failed to allocate extcon device\n"); + + err = devm_extcon_dev_register(dev, ev_edev); + if (err < 0) + dev_err(dev, "failed to register extcon device\n"); +#endif + + return 0; +} + +static int vehicle_dummy_hw_remove(struct platform_device *pdev) +{ + class_destroy(vehicle_dummy_class); + + return 0; +} + +static struct platform_driver vehicle_dummy_hw_driver = { + .probe = vehicle_dummy_hw_probe, + .remove = vehicle_dummy_hw_remove, + .driver = { + .name = "vehicle-dummy-hw", + } +}; + +static struct platform_device *dummy_pdev; + +static int vehicle_dummy_init(void) +{ + int err; + + dummy_pdev = platform_device_alloc("vehicle-dummy-hw", -1); + if (!dummy_pdev) { + pr_err("Failed to allocate dummy vehicle device\n"); + return -ENODEV; + } + + err = platform_device_add(dummy_pdev); + if (err != 0) { + pr_err("Failed to register dummy regulator device: %d\n", err); + platform_device_put(dummy_pdev); + return err; + } + + err = platform_driver_register(&vehicle_dummy_hw_driver); + if (err) { + pr_err("Failed to register dummy vehicle driver\n"); + platform_device_unregister(dummy_pdev); + return err; + } + return 0; +} + +static void __exit vehicle_dummy_exit(void) +{ + platform_driver_unregister(&vehicle_dummy_hw_driver); + platform_device_unregister(dummy_pdev); +} + +late_initcall(vehicle_dummy_init); +module_exit(vehicle_dummy_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("NXP Semiconductor"); +MODULE_DESCRIPTION("VEHICLE DUMMY HW"); diff --git a/drivers/misc/vehicle/vehicle_protocol_callback.c b/drivers/misc/vehicle/vehicle_protocol_callback.c new file mode 100644 index 000000000000..8a639a59656b --- /dev/null +++ b/drivers/misc/vehicle/vehicle_protocol_callback.c @@ -0,0 +1,336 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include "pb.h" +#include "pb_encode.h" +#include "pb_decode.h" +#include "vehicle_core.h" +#include "vehiclehalproto.pb.h" +#include "vehicle_protocol_callback.h" + +//VehiclePropertyType +#define VEHICLEPROPERTYTYPE_STRING 1048576 +#define VEHICLEPROPERTYTYPE_BOOLEAN 2097152 +#define VEHICLEPROPERTYTYPE_INT32 4194304 +#define VEHICLEPROPERTYTYPE_INT32_VEC 4259840 +#define VEHICLEPROPERTYTYPE_INT64 5242880 +#define VEHICLEPROPERTYTYPE_INT64_VEC 5308416 +#define VEHICLEPROPERTYTYPE_FLOAT 6291456 +#define VEHICLEPROPERTYTYPE_FLOAT_VEC 6356992 +#define VEHICLEPROPERTYTYPE_BYTES 7340032 +#define VEHICLEPROPERTYTYPE_MIXED 14680064 + + +bool decode_prop_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + emulator_VehiclePropGet propget; + + if (!pb_decode(stream, emulator_VehiclePropGet_fields, &propget)) { + pr_info("%s decode failed\n", __func__); + return false; + } + + return true; +} + +bool decode_config_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + emulator_VehiclePropConfig propconfigure; + + propconfigure.area_configs.funcs.decode = &decode_area_configs_callback; + propconfigure.config_array.funcs.decode = &decode_config_array_callback; + propconfigure.config_string.funcs.decode = &decode_config_string_callback; + + if (!pb_decode(stream, emulator_VehiclePropGet_fields, &propconfigure)) + return false; + + return true; +} + +bool decode_int32_values_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + uint64_t value; + + if (!pb_decode_svarint(stream, &value)) + return false; + *(u32 *)*arg = (u32)value; + return true; +} + +bool decode_int64_values_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + uint64_t value; + + if (!pb_decode_svarint(stream, &value)) + return false; + *(u64 *)*arg = (u64)value; + return true; +} + +bool decode_float_values_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + uint32_t value; + + if (!pb_decode_fixed32(stream, &value)) { + pr_info("float_values_callback failed\n"); + return false; + } + + *(u32 *)*arg = value; + return true; +} + +bool decode_value_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + struct vehicle_property_set *data = (struct vehicle_property_set *)(*arg); + emulator_VehiclePropValue propvalue; + + propvalue.int32_values.funcs.decode = &decode_int32_values_callback; + propvalue.int32_values.arg = &data->value; + propvalue.float_values.funcs.decode = &decode_float_values_callback; + propvalue.float_values.arg = &data->value; + propvalue.string_value.funcs.decode = &decode_config_string_callback; + propvalue.string_value.arg = &data->value; + propvalue.int64_values.funcs.decode = &decode_int64_values_callback; + propvalue.int64_values.arg = &data->value; + // Callback list must be extended if vhal starts to + // using new property variable type. + + if (!pb_decode(stream, emulator_VehiclePropValue_fields, &propvalue)) + return false; + + data->prop = propvalue.prop; + data->area_id = propvalue.area_id; + + return true; +} + +bool decode_area_configs_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + emulator_VehicleAreaConfig vehicle_area; + + if (!pb_decode(stream, emulator_VehicleAreaConfig_fields, &vehicle_area)) + return false; + return true; +} + +bool decode_config_array_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + uint64_t value; + + if (!pb_decode_varint(stream, &value)) + return false; + return true; +} + +bool decode_config_string_callback(pb_istream_t *stream, const pb_field_t *field, + void **arg) +{ + char *buffer; + + buffer = kmalloc(1024, GFP_KERNEL); + if (!buffer) { + pr_warn("Failed allocating memory for decode\n"); + return false; + } + + if (stream->bytes_left >= 1024) { + kfree(buffer); + return false; + } + + if (!pb_read(stream, buffer, stream->bytes_left)) { + kfree(buffer); + return false; + } + + kfree(buffer); + + return true; +} + +bool encode_prop_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + emulator_VehiclePropGet propget; + + //need fill arg emulator_VehiclePropGet if have this filed + if (!pb_encode(stream, emulator_VehiclePropGet_fields, &propget)) { + pr_info("%s encode failed\n", __func__); + return false; + } + + return true; +} + +bool encode_area_configs_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + emulator_VehicleAreaConfig vehicle_area; + + if (!pb_encode(stream, emulator_VehicleAreaConfig_fields, &vehicle_area)) + return false; + return true; +} + +bool encode_config_array_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_fixed32(stream, *arg); +} + +bool encode_config_string_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, *arg, strlen(*arg)); +} + +bool encode_config_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + emulator_VehiclePropConfig propconfigure; + + propconfigure.area_configs.funcs.encode = &encode_area_configs_callback; + propconfigure.config_array.funcs.encode = &encode_config_array_callback; + propconfigure.config_string.funcs.encode = &encode_config_string_callback; + + if (!pb_encode(stream, emulator_VehiclePropGet_fields, &propconfigure)) + return false; + + return true; +} + +bool encode_int32_values_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + u32 *value = (u32 *)*arg; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + + return pb_encode_svarint(stream, *value); +} + +bool encode_power_state_value_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + u32 *value = (u32 *)*arg; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!pb_encode_svarint(stream, *value)) + return false; + value++; + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!pb_encode_svarint(stream, *value)) + return false; + + return true; +} + +bool encode_int64_values_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + u64 *value = (u64 *)*arg; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + return pb_encode_svarint(stream, *value); +} + +bool encode_fix32_values_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + if (!pb_encode_tag_for_field(stream, field)) + return false; + return pb_encode_fixed32(stream, (u32 *)*arg); +} + +bool encode_power_state_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + struct vehicle_power_req *data = (struct vehicle_power_req *)(*arg); + emulator_VehiclePropValue propvalue = {}; + + propvalue.prop = data->prop; + propvalue.has_value_type = true; + propvalue.has_timestamp = true; + propvalue.timestamp = 0; +#ifndef CONFIG_VEHICLE_DRIVER_OREO + propvalue.has_status = true; + propvalue.status = 0; +#endif + propvalue.int32_values.funcs.encode = &encode_power_state_value_callback; + propvalue.int32_values.arg = &data->state; + propvalue.value_type = VEHICLEPROPERTYTYPE_INT32; + + pb_encode_tag_for_field(stream, field); + if (!pb_encode_submessage(stream, emulator_VehiclePropValue_fields, &propvalue)) { + pr_info("%s encode submessage failed\n", __func__); + return false; + } + return true; +} + +bool encode_value_callback(pb_ostream_t *stream, const pb_field_t *field, + void * const *arg) +{ + struct vehicle_property_set *data = (struct vehicle_property_set *)(*arg); + emulator_VehiclePropValue propvalue = {}; + + propvalue.prop = data->prop; + propvalue.has_value_type = true; + propvalue.has_timestamp = true; + propvalue.timestamp = 0; + propvalue.has_area_id = true; + propvalue.area_id = data->area_id; +#ifndef CONFIG_VEHICLE_DRIVER_OREO + propvalue.has_status = true; + propvalue.status = 0; +#endif + if (data->prop == HVAC_TEMPERATURE_SET) { + propvalue.float_values.funcs.encode = &encode_fix32_values_callback; + propvalue.float_values.arg = &data->value; + propvalue.value_type = VEHICLEPROPERTYTYPE_FLOAT; + } else if (HVAC_FAN_SPEED == data->prop || HVAC_FAN_DIRECTION == data->prop || + HVAC_AUTO_ON == data->prop || HVAC_AC_ON == data->prop || + HVAC_RECIRC_ON == data->prop || HVAC_POWER_ON == data->prop || + HVAC_DEFROSTER == data->prop || TURN_SIGNAL_STATE == data->prop || + GEAR_SELECTION == data->prop || HVAC_SEAT_TEMPERATURE == data->prop) { + propvalue.int32_values.funcs.encode = &encode_int32_values_callback; + propvalue.int32_values.arg = &data->value; + propvalue.value_type = VEHICLEPROPERTYTYPE_INT32; + } + + //propvalue.int64_values.funcs.encode = &encode_int64_values_callback; + //propvalue.string_value.funcs.encode = &encode_config_string_callback; + //propvalue.bytes_value.funcs.encode = &encode_config_string_callback; + pb_encode_tag_for_field(stream, field); + if (!pb_encode_submessage(stream, emulator_VehiclePropValue_fields, &propvalue)) { + pr_info("%s encode submessage failed\n", __func__); + return false; + } + return true; +} + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("NXP Semiconductor"); + diff --git a/drivers/misc/vehicle/vehicle_protocol_callback.h b/drivers/misc/vehicle/vehicle_protocol_callback.h new file mode 100644 index 000000000000..0610781471d8 --- /dev/null +++ b/drivers/misc/vehicle/vehicle_protocol_callback.h @@ -0,0 +1,79 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef VEHICLE_PROTOCOL_CALLBACK_H_ +#define VEHICLE_PROTOCOL_CALLBACK_H_ + +bool decode_prop_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_config_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_value_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_config_string_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_config_array_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_area_configs_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_float_values_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_int64_values_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool decode_int32_values_callback(pb_istream_t *stream, + const pb_field_t *field, + void **arg); +bool encode_value_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_prop_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_area_configs_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_config_array_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_config_string_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_config_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_int32_values_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_int64_values_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_fix32_values_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_power_state_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); +bool encode_power_state_value_callback(pb_ostream_t *stream, + const pb_field_t *field, + void * const *arg); + +#endif diff --git a/drivers/misc/vehicle/vehiclehalproto.pb.c b/drivers/misc/vehicle/vehiclehalproto.pb.c new file mode 100644 index 000000000000..3face5a2179d --- /dev/null +++ b/drivers/misc/vehicle/vehiclehalproto.pb.c @@ -0,0 +1,166 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include "vehiclehalproto.pb.h" + +const pb_field_t emulator_VehicleAreaConfig_fields[8] = { + PB_FIELD2(1, INT32, REQUIRED, STATIC, FIRST, + emulator_VehicleAreaConfig, area_id, area_id, 0), + PB_FIELD2(2, SINT32, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, min_int32_value, area_id, 0), + PB_FIELD2(3, SINT32, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, max_int32_value, min_int32_value, 0), + PB_FIELD2(4, SINT64, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, min_int64_value, max_int32_value, 0), + PB_FIELD2(5, SINT64, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, max_int64_value, min_int64_value, 0), + PB_FIELD2(6, FLOAT, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, min_float_value, max_int64_value, 0), + PB_FIELD2(7, FLOAT, OPTIONAL, STATIC, OTHER, + emulator_VehicleAreaConfig, max_float_value, min_float_value, 0), + PB_LAST_FIELD +}; + +const pb_field_t emulator_VehiclePropConfig_fields[12] = { + PB_FIELD2(1, INT32, REQUIRED, STATIC, FIRST, + emulator_VehiclePropConfig, prop, prop, 0), + PB_FIELD2(2, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, access, prop, 0), + PB_FIELD2(3, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, change_mode, access, 0), + PB_FIELD2(4, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, value_type, change_mode, 0), + PB_FIELD2(5, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, supported_areas, value_type, 0), + PB_FIELD2(6, MESSAGE, REPEATED, CALLBACK, OTHER, emulator_VehiclePropConfig, + area_configs, supported_areas, &emulator_VehicleAreaConfig_fields), + PB_FIELD2(7, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, config_flags, area_configs, 0), + PB_FIELD2(8, INT32, REPEATED, CALLBACK, OTHER, + emulator_VehiclePropConfig, config_array, config_flags, 0), + PB_FIELD2(9, STRING, OPTIONAL, CALLBACK, OTHER, + emulator_VehiclePropConfig, config_string, config_array, 0), + PB_FIELD2(10, FLOAT, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, min_sample_rate, config_string, 0), + PB_FIELD2(11, FLOAT, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropConfig, max_sample_rate, min_sample_rate, 0), + PB_LAST_FIELD +}; + +#ifdef CONFIG_VEHICLE_DRIVER_OREO +const pb_field_t emulator_VehiclePropValue_fields[10] = { + PB_FIELD2(1, INT32, REQUIRED, STATIC, FIRST, + emulator_VehiclePropValue, prop, prop, 0), + PB_FIELD2(2, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropValue, value_type, prop, 0), + PB_FIELD2(3, INT64, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropValue, timestamp, value_type, 0), + PB_FIELD2(4, INT32, OPTIONAL, STATIC, OTHER, + emulator_VehiclePropValue, area_id, timestamp, 0), + PB_FIELD2(5, SINT32, REPEATED, CALLBACK, OTHER, + emulator_VehiclePropValue, int32_values, area_id, 0), + PB_FIELD2(6, SINT64, REPEATED, CALLBACK, OTHER, + emulator_VehiclePropValue, int64_values, int32_values, 0), + PB_FIELD2(7, FLOAT, REPEATED, CALLBACK, OTHER, + emulator_VehiclePropValue, float_values, int64_values, 0), + PB_FIELD2(8, STRING, OPTIONAL, CALLBACK, OTHER, + emulator_VehiclePropValue, string_value, float_values, 0), + PB_FIELD2(9, BYTES, OPTIONAL, CALLBACK, OTHER, + emulator_VehiclePropValue, bytes_value, string_value, 0), + PB_LAST_FIELD +}; +#else +const pb_field_t emulator_VehiclePropValue_fields[11] = { + PB_FIELD2(1, INT32, REQUIRED, STATIC, FIRST, emulator_VehiclePropValue, + prop, prop, 0), + PB_FIELD2(2, INT32, OPTIONAL, STATIC, OTHER, emulator_VehiclePropValue, + value_type, prop, 0), + PB_FIELD2(3, INT64, OPTIONAL, STATIC, OTHER, emulator_VehiclePropValue, + timestamp, value_type, 0), + PB_FIELD2(4, INT32, OPTIONAL, STATIC, OTHER, emulator_VehiclePropValue, + area_id, timestamp, 0), + PB_FIELD2(5, SINT32, REPEATED, CALLBACK, OTHER, emulator_VehiclePropValue, + int32_values, area_id, 0), + PB_FIELD2(6, SINT64, REPEATED, CALLBACK, OTHER, emulator_VehiclePropValue, + int64_values, int32_values, 0), + PB_FIELD2(7, FLOAT, REPEATED, CALLBACK, OTHER, emulator_VehiclePropValue, + float_values, int64_values, 0), + PB_FIELD2(8, STRING, OPTIONAL, CALLBACK, OTHER, emulator_VehiclePropValue, + string_value, float_values, 0), + PB_FIELD2(9, BYTES, OPTIONAL, CALLBACK, OTHER, emulator_VehiclePropValue, + bytes_value, string_value, 0), + PB_FIELD2(10, ENUM, OPTIONAL, STATIC, OTHER, emulator_VehiclePropValue, + status, bytes_value, 0), + PB_LAST_FIELD +}; +#endif + +const pb_field_t emulator_VehiclePropGet_fields[3] = { + PB_FIELD2(1, INT32, REQUIRED, STATIC, FIRST, emulator_VehiclePropGet, + prop, prop, 0), + PB_FIELD2(2, INT32, OPTIONAL, STATIC, OTHER, emulator_VehiclePropGet, + area_id, prop, 0), + PB_LAST_FIELD +}; + +const pb_field_t emulator_EmulatorMessage_fields[6] = { + PB_FIELD2(1, ENUM, REQUIRED, STATIC, FIRST, emulator_EmulatorMessage, msg_type, + msg_type, 0), + PB_FIELD2(2, ENUM, OPTIONAL, STATIC, OTHER, emulator_EmulatorMessage, status, + msg_type, 0), + PB_FIELD2(3, MESSAGE, REPEATED, CALLBACK, OTHER, emulator_EmulatorMessage, + prop, status, &emulator_VehiclePropGet_fields), + PB_FIELD2(4, MESSAGE, REPEATED, CALLBACK, OTHER, emulator_EmulatorMessage, + config, prop, &emulator_VehiclePropConfig_fields), + PB_FIELD2(5, MESSAGE, REPEATED, CALLBACK, OTHER, emulator_EmulatorMessage, + value, config, &emulator_VehiclePropValue_fields), + PB_LAST_FIELD +}; + +/* Check that field information fits in pb_field_t */ +#if !defined(PB_FIELD_32BIT) +/* If you get an error here, it means that you need to define PB_FIELD_32BIT + * compile-time option. You can do that in pb.h or on compiler command line. + * + * The reason you need to do this is that some of your messages contain tag + * numbers or field sizes that are larger than what can fit in 8 or 16 bit + * field descriptors. + */ +STATIC_ASSERT((pb_membersize(emulator_VehiclePropConfig, area_configs) < 65536 +&& pb_membersize(emulator_EmulatorMessage, prop) < 65536 +&& pb_membersize(emulator_EmulatorMessage, config) < 65536 +&& pb_membersize(emulator_EmulatorMessage, value) < 65536), +YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_emulator_VehicleAreaConfig_emulator_\ +VehiclePropConfig_emulator_VehiclePropValue_emulator_VehiclePropGet_emulator_EmulatorMessage) +#endif + +#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) +/* If you get an error here, it means that you need to define PB_FIELD_16BIT + * compile-time option. You can do that in pb.h or on compiler command line. + * + * The reason you need to do this is that some of your messages contain tag + * numbers or field sizes that are larger than what can fit in the default + * 8 bit descriptors. + */ +STATIC_ASSERT((pb_membersize(emulator_VehiclePropConfig, area_configs) < 256 +&& pb_membersize(emulator_EmulatorMessage, prop) < 256 +&& pb_membersize(emulator_EmulatorMessage, config) < 256 +&& pb_membersize(emulator_EmulatorMessage, value) < 256), +YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_emulator_VehicleAreaConfig_emulator_\ +VehiclePropConfig_emulator_VehiclePropValue_emulator_VehiclePropGet_emulator_EmulatorMessage) +#endif + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("NXP Semiconductor"); diff --git a/drivers/misc/vehicle/vehiclehalproto.pb.h b/drivers/misc/vehicle/vehiclehalproto.pb.h new file mode 100644 index 000000000000..90e981bd06f7 --- /dev/null +++ b/drivers/misc/vehicle/vehiclehalproto.pb.h @@ -0,0 +1,191 @@ +/* + * Copyright 2018 NXP + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _PB_VEHICLEHALPROTO_PB_H_ +#define _PB_VEHICLEHALPROTO_PB_H_ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Enum definitions */ +typedef enum _emulator_MsgType { + emulator_MsgType_GET_CONFIG_CMD = 0, + emulator_MsgType_GET_CONFIG_RESP = 1, + emulator_MsgType_GET_CONFIG_ALL_CMD = 2, + emulator_MsgType_GET_CONFIG_ALL_RESP = 3, + emulator_MsgType_GET_PROPERTY_CMD = 4, + emulator_MsgType_GET_PROPERTY_RESP = 5, + emulator_MsgType_GET_PROPERTY_ALL_CMD = 6, + emulator_MsgType_GET_PROPERTY_ALL_RESP = 7, + emulator_MsgType_SET_PROPERTY_CMD = 8, + emulator_MsgType_SET_PROPERTY_RESP = 9, + emulator_MsgType_SET_PROPERTY_ASYNC = 10 +} emulator_MsgType; + +typedef enum _emulator_Status { + emulator_Status_RESULT_OK = 0, + emulator_Status_ERROR_UNKNOWN = 1, + emulator_Status_ERROR_UNIMPLEMENTED_CMD = 2, + emulator_Status_ERROR_INVALID_PROPERTY = 3, + emulator_Status_ERROR_INVALID_AREA_ID = 4, + emulator_Status_ERROR_PROPERTY_UNINITIALIZED = 5, + emulator_Status_ERROR_WRITE_ONLY_PROPERTY = 6, + emulator_Status_ERROR_MEMORY_ALLOC_FAILED = 7, + emulator_Status_ERROR_INVALID_OPERATION = 8 +} emulator_Status; + +#ifndef CONFIG_VEHICLE_DRIVER_OREO +typedef enum _emulator_VehiclePropStatus { + emulator_VehiclePropStatus_AVAILABLE = 0, + emulator_VehiclePropStatus_UNAVAILABLE = 1, + emulator_VehiclePropStatus_ERROR = 2 +} emulator_VehiclePropStatus; +#endif + +/* Struct definitions */ +typedef struct _emulator_VehicleAreaConfig { + int32_t area_id; + bool has_min_int32_value; + int32_t min_int32_value; + bool has_max_int32_value; + int32_t max_int32_value; + bool has_min_int64_value; + int64_t min_int64_value; + bool has_max_int64_value; + int64_t max_int64_value; + bool has_min_float_value; + float min_float_value; + bool has_max_float_value; + float max_float_value; +} emulator_VehicleAreaConfig; + +typedef struct _emulator_VehiclePropGet { + int32_t prop; + bool has_area_id; + int32_t area_id; +} emulator_VehiclePropGet; + +typedef struct _emulator_VehiclePropValue { + int32_t prop; + bool has_value_type; + int32_t value_type; + bool has_timestamp; + int64_t timestamp; + bool has_area_id; + int32_t area_id; + pb_callback_t int32_values; + pb_callback_t int64_values; + pb_callback_t float_values; + pb_callback_t string_value; + pb_callback_t bytes_value; +#ifndef CONFIG_VEHICLE_DRIVER_OREO + bool has_status; + emulator_VehiclePropStatus status; +#endif +} emulator_VehiclePropValue; + +typedef struct _emulator_VehiclePropConfig { + int32_t prop; + bool has_access; + int32_t access; + bool has_change_mode; + int32_t change_mode; + bool has_value_type; + int32_t value_type; + bool has_supported_areas; + int32_t supported_areas; + pb_callback_t area_configs; + bool has_config_flags; + int32_t config_flags; + pb_callback_t config_array; + pb_callback_t config_string; + bool has_min_sample_rate; + float min_sample_rate; + bool has_max_sample_rate; + float max_sample_rate; +} emulator_VehiclePropConfig; + +typedef struct _emulator_EmulatorMessage { + emulator_MsgType msg_type; + bool has_status; + emulator_Status status; + pb_callback_t prop; + pb_callback_t config; + pb_callback_t value; +} emulator_EmulatorMessage; + +/* Default values for struct fields */ + +/* Field tags (for use in manual encoding/decoding) */ +#define emulator_VehicleAreaConfig_area_id_tag 1 +#define emulator_VehicleAreaConfig_min_int32_value_tag 2 +#define emulator_VehicleAreaConfig_max_int32_value_tag 3 +#define emulator_VehicleAreaConfig_min_int64_value_tag 4 +#define emulator_VehicleAreaConfig_max_int64_value_tag 5 +#define emulator_VehicleAreaConfig_min_float_value_tag 6 +#define emulator_VehicleAreaConfig_max_float_value_tag 7 +#define emulator_VehiclePropGet_prop_tag 1 +#define emulator_VehiclePropGet_area_id_tag 2 +#define emulator_VehiclePropValue_prop_tag 1 +#define emulator_VehiclePropValue_value_type_tag 2 +#define emulator_VehiclePropValue_timestamp_tag 3 +#ifndef CONFIG_VEHICLE_DRIVER_OREO +#define emulator_VehiclePropValue_status_tag 10 +#endif +#define emulator_VehiclePropValue_area_id_tag 4 +#define emulator_VehiclePropValue_int32_values_tag 5 +#define emulator_VehiclePropValue_int64_values_tag 6 +#define emulator_VehiclePropValue_float_values_tag 7 +#define emulator_VehiclePropValue_string_value_tag 8 +#define emulator_VehiclePropValue_bytes_value_tag 9 +#define emulator_VehiclePropConfig_prop_tag 1 +#define emulator_VehiclePropConfig_access_tag 2 +#define emulator_VehiclePropConfig_change_mode_tag 3 +#define emulator_VehiclePropConfig_value_type_tag 4 +#define emulator_VehiclePropConfig_supported_areas_tag 5 +#define emulator_VehiclePropConfig_area_configs_tag 6 +#define emulator_VehiclePropConfig_config_flags_tag 7 +#define emulator_VehiclePropConfig_config_array_tag 8 +#define emulator_VehiclePropConfig_config_string_tag 9 +#define emulator_VehiclePropConfig_min_sample_rate_tag 10 +#define emulator_VehiclePropConfig_max_sample_rate_tag 11 +#define emulator_EmulatorMessage_msg_type_tag 1 +#define emulator_EmulatorMessage_status_tag 2 +#define emulator_EmulatorMessage_prop_tag 3 +#define emulator_EmulatorMessage_config_tag 4 +#define emulator_EmulatorMessage_value_tag 5 + +/* Struct field encoding specification for nanopb */ +extern const pb_field_t emulator_VehicleAreaConfig_fields[8]; +extern const pb_field_t emulator_VehiclePropConfig_fields[12]; +#ifdef CONFIG_VEHICLE_DRIVER_OREO +extern const pb_field_t emulator_VehiclePropValue_fields[10]; +#else +extern const pb_field_t emulator_VehiclePropValue_fields[11]; +#endif +extern const pb_field_t emulator_VehiclePropGet_fields[3]; +extern const pb_field_t emulator_EmulatorMessage_fields[6]; + +/* Maximum encoded size of messages (where known) */ +#define emulator_VehicleAreaConfig_size 55 +#define emulator_VehiclePropGet_size 22 + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif From 7a1c10a36a3a2be5f01fbca096225d6846ff5b4b Mon Sep 17 00:00:00 2001 From: Sugar Zhang Date: Wed, 18 Jun 2025 10:46:07 +0800 Subject: [PATCH 37/49] arm64: dts: rockchip: Add rv1126bp dtsi Signed-off-by: Sugar Zhang Change-Id: I9cdc728f571ff10cf65e96e18d4cecb3dc564140 --- arch/arm64/boot/dts/rockchip/rv1126bp.dtsi | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 arch/arm64/boot/dts/rockchip/rv1126bp.dtsi diff --git a/arch/arm64/boot/dts/rockchip/rv1126bp.dtsi b/arch/arm64/boot/dts/rockchip/rv1126bp.dtsi new file mode 100644 index 000000000000..f4e57e04e3d8 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rv1126bp.dtsi @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2025 Rockchip Electronics Co., Ltd. + */ + +#include "rv1126b.dtsi" + +/ { +}; From 251eae10868dc3f8fdda4e4bac443b7deb768347 Mon Sep 17 00:00:00 2001 From: Sugar Zhang Date: Wed, 18 Jun 2025 10:46:41 +0800 Subject: [PATCH 38/49] arm64: dts: rockchip: rv1126bp-evb: Include rv1126bp dtsi Signed-off-by: Sugar Zhang Change-Id: Ia55b97d05a4a20fb4789ccd7495010353281d505 --- arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14-dual-cam.dts | 2 +- arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14.dts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14-dual-cam.dts b/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14-dual-cam.dts index 0f06f0ea6252..4a69dd9b6c7f 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14-dual-cam.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14-dual-cam.dts @@ -4,7 +4,7 @@ */ /dts-v1/; -#include "rv1126b.dtsi" +#include "rv1126bp.dtsi" #include "rv1126bp-evb.dtsi" #include "rv1126bp-evb-v14.dtsi" #include "rv1126bp-evb-v14-dual-cam.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14.dts b/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14.dts index 89c248f9eb05..1c97ab42c895 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126bp-evb-v14.dts @@ -4,7 +4,7 @@ */ /dts-v1/; -#include "rv1126b.dtsi" +#include "rv1126bp.dtsi" #include "rv1126bp-evb.dtsi" #include "rv1126bp-evb-v14.dtsi" #include "rv1126bp-evb-cam-csi0.dtsi" From 19e54153f8078824c5d21bac1a1ca230cb123462 Mon Sep 17 00:00:00 2001 From: Chaoyi Chen Date: Thu, 12 Jun 2025 03:16:12 +0000 Subject: [PATCH 39/49] drm/rockchip: rgb: Add support for set data map mode The RV1126B introduces a data map mechanism, allowing MCU/BT1120/BT656 pin data map data to be set. This makes pin connections compatible with both earlier and new platforms. Change-Id: I5acbe9e36116c07857c808ad7de1dd14b1214e04 Signed-off-by: Chaoyi Chen --- drivers/gpu/drm/rockchip/rockchip_drm_drv.h | 1 + drivers/gpu/drm/rockchip/rockchip_rgb.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h index 6d0d223ee17c..665f65181963 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h +++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h @@ -274,6 +274,7 @@ struct rockchip_crtc_state { int output_mode; int output_bpc; int output_flags; + int data_map_mode; bool enable_afbc; /** * @splice_mode: enabled when display a hdisplay > 4096 on rk3588 diff --git a/drivers/gpu/drm/rockchip/rockchip_rgb.c b/drivers/gpu/drm/rockchip/rockchip_rgb.c index 86a53924a7c9..89e48f4f2a4d 100644 --- a/drivers/gpu/drm/rockchip/rockchip_rgb.c +++ b/drivers/gpu/drm/rockchip/rockchip_rgb.c @@ -144,6 +144,7 @@ struct rockchip_rgb { u8 id; u32 max_dclk_rate; u32 mcu_pix_total; + int data_map_mode; struct device *dev; struct device_node *np_mcu_panel; struct drm_panel *panel; @@ -391,6 +392,7 @@ rockchip_rgb_encoder_atomic_check(struct drm_encoder *encoder, break; } + s->data_map_mode = rgb->data_map_mode; s->output_type = DRM_MODE_CONNECTOR_DPI; s->bus_flags = info->bus_flags; s->tv_state = &conn_state->tv; @@ -1060,6 +1062,10 @@ static int rockchip_rgb_probe(struct platform_device *pdev) id = 0; rgb->data_sync_bypass = of_property_read_bool(dev->of_node, "rockchip,data-sync-bypass"); + if (of_property_read_u32(dev->of_node, "rockchip,data-map-mode", &rgb->data_map_mode)) + rgb->data_map_mode = -1; + if (rgb->data_map_mode < 0 || rgb->data_map_mode > 3) + rgb->data_map_mode = -1; fwnode_mcu_panel = device_get_named_child_node(dev, "mcu-panel"); if (fwnode_mcu_panel) { From 885440243620d3ecf6614aeffe308eab3f319c31 Mon Sep 17 00:00:00 2001 From: Chaoyi Chen Date: Thu, 12 Jun 2025 03:19:07 +0000 Subject: [PATCH 40/49] drm/rockchip: vop: Add support for set data map mode for RV1126B The RV1126B introduce data map mode for BT1120/BT656/MCU. MCU: - 1'b0: mcu_data[23:0] - 1'b1: {mcu_data[7:3],3'b0,mcu_data[2:0], mcu_data[7:5],2'b0,mcu_data[4:0],3'b0} BT1120: - 2'b00: {8'd0,bt1120[15:0]} - 2'b01: {bt1120[15:0],8'd0} - 2'b10: reserved - 2'b11: {bt1120[15:11],3'd0,bt1120[10:5],2'd0,bt1120[4:0],3'b0} BT656: - 2'b00: {16'd0,bt656[7:0]} - 2'b01: {8'd0,bt656[7:0],8'd0} - 2'b10: {bt656[7:0],16'd0} - 2'b11: {bt656[7:3],3'd0,bt656[2:0],bt656[7:5],2'd0,bt656[4:0],3'd0} Change-Id: I875a2675004c25a3382135af337c2fb82c0d2563 Signed-off-by: Chaoyi Chen --- drivers/gpu/drm/rockchip/rockchip_drm_vop.c | 9 +++++++++ drivers/gpu/drm/rockchip/rockchip_drm_vop.h | 3 +++ drivers/gpu/drm/rockchip/rockchip_vop_reg.c | 3 +++ 3 files changed, 15 insertions(+) diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c index 5c3f82df2bac..9eac6c767165 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c @@ -4176,6 +4176,15 @@ static void vop_crtc_atomic_enable(struct drm_crtc *crtc, if (vop->lut_active) vop_crtc_load_lut(crtc); + if (s->data_map_mode != -1) { + if (s->output_if & VOP_OUTPUT_IF_BT1120) + VOP_CTRL_SET(vop, bt1120_data_map_mode, s->data_map_mode); + else if (s->output_if & VOP_OUTPUT_IF_BT656) + VOP_CTRL_SET(vop, bt656_data_map_mode, s->data_map_mode); + else if (vop->mcu_timing.mcu_pix_total) + VOP_CTRL_SET(vop, mcu_data_map_mode, s->data_map_mode); + } + if (vop->mcu_timing.mcu_pix_total) { /* * For RK3576_LITE/RK3506/RV1126B diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h index 5f5ea7ccfcac..e2b26d7c7c1b 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h @@ -458,14 +458,17 @@ struct vop_ctrl { struct vop_reg mcu_type; struct vop_reg mcu_rw_bypass_port; struct vop_reg mcu_force_rdn; + struct vop_reg mcu_data_map_mode; /* bt1120 */ struct vop_reg bt1120_uv_swap; struct vop_reg bt1120_yc_swap; struct vop_reg bt1120_en; + struct vop_reg bt1120_data_map_mode; /* bt656 */ struct vop_reg bt656_en; + struct vop_reg bt656_data_map_mode; struct vop_reg reg_done_frm; struct vop_reg cfg_done; diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c index b882d8f020ab..cbd3431d2586 100644 --- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c +++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c @@ -1946,6 +1946,9 @@ static const struct vop_ctrl rv1126b_ctrl_data = { .bt656_en = VOP_REG(RK3366_LIT_DSP_CTRL0, 0x1, 6), .mcu_force_rdn = VOP_REG(RV1126B_DSP_CTRL1, 0x1, 21), + .mcu_data_map_mode = VOP_REG(RV1126B_DSP_CTRL1, 0x1, 20), + .bt1120_data_map_mode = VOP_REG(RV1126B_DSP_CTRL1, 0x3, 18), + .bt656_data_map_mode = VOP_REG(RV1126B_DSP_CTRL1, 0x3, 16), .dsp_interlace = VOP_REG(RK3366_LIT_DSP_CTRL2, 0x1, 0), .dither_up_en = VOP_REG(RK3366_LIT_DSP_CTRL2, 0x1, 2), From 1dbf2d1f5ed02b7be2500413c9248ad5f1ef8fdf Mon Sep 17 00:00:00 2001 From: Chaoyi Chen Date: Fri, 13 Jun 2025 01:46:11 +0000 Subject: [PATCH 41/49] dt-bindings: display: Add rockchip MCU/BT1120/BT656 data map Change-Id: Ib38aa21f2752026bf395130620c2f279bfea2f74 Signed-off-by: Chaoyi Chen --- include/dt-bindings/display/rockchip_vop.h | 100 +++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/include/dt-bindings/display/rockchip_vop.h b/include/dt-bindings/display/rockchip_vop.h index 55d25bfd8875..f785d8a5bca4 100644 --- a/include/dt-bindings/display/rockchip_vop.h +++ b/include/dt-bindings/display/rockchip_vop.h @@ -20,4 +20,104 @@ #define ROCKCHIP_VOP2_PHY_ID_INVALID -1 +/* mcu_data[23:0] */ +#define ROCKCHIP_MCU_DATA_MAP_DATA_1x24 0 +/* + * { mcu_data[7:3], 3'b0, mcu_data[2:0], mcu_data[7:5], 2'b0, mcu_data[4:0], 3'b0 } + * That is: + * LCDC_D23 -> MCU_D7_M1 + * LCDC_D22 -> MCU_D6_M1 + * LCDC_D21 -> MCU_D5_M1 + * LCDC_D20 -> MCU_D4_M1 + * LCDC_D19 -> MCU_D3_M1 + * LCDC_D18 -> N/A + * LCDC_D17 -> N/A + * LCDC_D16 -> N/A + * LCDC_D15 -> MCU_D2_M1 + * LCDC_D14 -> MCU_D1_M1 + * LCDC_D13 -> MCU_D0_M1 + * LCDC_D12 -> MCU_D7_M0 + * LCDC_D11 -> MCU_D6_M0 + * LCDC_D10 -> MCU_D5_M0 + * LCDC_D9 -> N/A + * LCDC_D8 -> N/A + * LCDC_D7 -> MCU_D4_M0 + * LCDC_D6 -> MCU_D3_M0 + * LCDC_D5 -> MCU_D2_M0 + * LCDC_D4 -> MCU_D1_M0 + * LCDC_D3 -> MCU_D0_M0 + * LCDC_D2 -> N/A + * LCDC_D1 -> N/A + * LCDC_D0 -> N/A + */ +#define ROCKCHIP_MCU_DATA_MAP_DATA_WITH_DUMMY_0 1 +/* { 8'd0, BT1120[15:0] } */ +#define ROCKCHIP_BT1120_DATA_MAP_DATA_PADLO 0 +/* { BT1120[15:0], 8'd0 }*/ +#define ROCKCHIP_BT1120_DATA_MAP_DATA_PADHI 1 +/* + * { BT1120[15:11], 3'd0, BT1120[10:5], 2'd0, BT1120[4:0], 3'b0 } + * That is: + * LCDC_D23 -> BT1120_D15 + * LCDC_D22 -> BT1120_D14 + * LCDC_D21 -> BT1120_D13 + * LCDC_D20 -> BT1120_D12 + * LCDC_D19 -> BT1120_D11 + * LCDC_D18 -> N/A + * LCDC_D17 -> N/A + * LCDC_D16 -> N/A + * LCDC_D15 -> BT1120_D10 + * LCDC_D14 -> BT1120_D9 + * LCDC_D13 -> BT1120_D8 + * LCDC_D12 -> BT1120_D7 + * LCDC_D11 -> BT1120_D6 + * LCDC_D10 -> BT1120_D5 + * LCDC_D9 -> N/A + * LCDC_D8 -> N/A + * LCDC_D7 -> BT1120_D4 + * LCDC_D6 -> BT1120_D3 + * LCDC_D5 -> BT1120_D2 + * LCDC_D4 -> BT1120_D1 + * LCDC_D3 -> BT1120_D0 + * LCDC_D2 -> N/A + * LCDC_D1 -> N/A + * LCDC_D0 -> N/A + */ +#define ROCKCHIP_BT1120_DATA_MAP_DATA_WITH_DUMMY_0 3 +/* { 16'd0, BT656[7:0] }*/ +#define ROCKCHIP_BT656_DATA_MAP_PADLO 0 +/* { 8'd0, BT656[7:0], 8'd0 }*/ +#define ROCKCHIP_BT656_DATA_MAP_DUMMY_HI_LO 1 +/* { BT656[7:0], 16'd0 }*/ +#define ROCKCHIP_BT656_DATA_MAP_PADHI 2 +/* + * { BT656[7:3], 3'd0, BT656[2:0], BT656[7:5], 2'd0, BT656[4:0], 3'd0 } + * That is: + * LCDC_D23 -> BT656_D7_M1 + * LCDC_D22 -> BT656_D6_M1 + * LCDC_D21 -> BT656_D5_M1 + * LCDC_D20 -> BT656_D4_M1 + * LCDC_D19 -> BT656_D3_M1 + * LCDC_D18 -> N/A + * LCDC_D17 -> N/A + * LCDC_D16 -> N/A + * LCDC_D15 -> BT656_D2_M1 + * LCDC_D14 -> BT656_D1_M1 + * LCDC_D13 -> BT656_D0_M1 + * LCDC_D12 -> BT656_D7_M0 + * LCDC_D11 -> BT656_D6_M0 + * LCDC_D10 -> BT656_D5_M0 + * LCDC_D9 -> N/A + * LCDC_D8 -> N/A + * LCDC_D7 -> BT656_D4_M0 + * LCDC_D6 -> BT656_D3_M0 + * LCDC_D5 -> BT656_D2_M0 + * LCDC_D4 -> BT656_D1_M0 + * LCDC_D3 -> BT656_D0_M0 + * LCDC_D2 -> N/A + * LCDC_D1 -> N/A + * LCDC_D0 -> N/A + */ +#define ROCKCHIP_BT656_DATA_MAP_DATA_WITH_DUMMY_0 3 + #endif From c54e57f476c19823bbba4f1ca692091dee21ec77 Mon Sep 17 00:00:00 2001 From: Chaoyi Chen Date: Thu, 12 Jun 2025 03:52:26 +0000 Subject: [PATCH 42/49] arm64: dts: rockchip: rv1126bp-evb: Make rgb output compatible with RV1126 Change-Id: I346c746a18454bf2e912ad777edcd55d25525690 Signed-off-by: Chaoyi Chen --- arch/arm64/boot/dts/rockchip/rv1126bp-evb.dtsi | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/arch/arm64/boot/dts/rockchip/rv1126bp-evb.dtsi b/arch/arm64/boot/dts/rockchip/rv1126bp-evb.dtsi index 81acd65cf9cf..ec31468906b5 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126bp-evb.dtsi +++ b/arch/arm64/boot/dts/rockchip/rv1126bp-evb.dtsi @@ -383,6 +383,17 @@ status = "okay"; }; +&rgb { + /* + * RV1126 compatible pin output mode 0. + * + * MCU: ROCKCHIP_MCU_DATA_MAP_DATA_1x24 + * BT1120: ROCKCHIP_BT1120_DATA_MAP_DATA_PADLO + * BT656: ROCKCHIP_BT656_DATA_MAP_PADLO + */ + rockchip,data-map-mode = <0>; +}; + &rkdvbm { status = "okay"; }; From 4940ad2c423e503269dc440b80e663ed889f7d87 Mon Sep 17 00:00:00 2001 From: Chaoyi Chen Date: Thu, 12 Jun 2025 07:41:35 +0000 Subject: [PATCH 43/49] arm64: dts: rockchip: rv1126b-evb2: Add explicit data-map-mode for RGB output Change-Id: I772479830d546cfcf2416f76191685b7f09cef32 Signed-off-by: Chaoyi Chen --- .../boot/dts/rockchip/rv1126b-evb2-v10-mcu-k350c4516t.dts | 2 ++ .../rockchip/rv1126b-evb2-v10-sii9022-bt1120-to-hdmi.dts | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-mcu-k350c4516t.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-mcu-k350c4516t.dts index baa7b7a71285..9414553a06f3 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-mcu-k350c4516t.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-mcu-k350c4516t.dts @@ -6,6 +6,7 @@ /dts-v1/; #include +#include #include "rv1126b-evb2-v10.dts" / { @@ -15,6 +16,7 @@ &rgb { status = "okay"; + rockchip,data-map-mode = ; rockchip,data-sync-bypass; pinctrl-names = "default"; /* diff --git a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-sii9022-bt1120-to-hdmi.dts b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-sii9022-bt1120-to-hdmi.dts index 7ffacffa6ab3..d05397b88aab 100644 --- a/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-sii9022-bt1120-to-hdmi.dts +++ b/arch/arm64/boot/dts/rockchip/rv1126b-evb2-v10-sii9022-bt1120-to-hdmi.dts @@ -6,6 +6,7 @@ /dts-v1/; #include +#include #include "rv1126b-evb2-v10.dts" / { @@ -60,6 +61,11 @@ &rgb { status = "okay"; + /* + * BT1120: ROCKCHIP_BT1120_DATA_MAP_DATA_WITH_DUMMY_0 + * BT656: ROCKCHIP_BT656_DATA_MAP_DATA_WITH_DUMMY_0 + */ + rockchip,data-map-mode = ; pinctrl-names = "default"; /* * bt1120_pins for bt1120 From fb91362a236886e1956af3ae7cff13173fe0ec80 Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Tue, 22 Apr 2025 11:52:39 +0800 Subject: [PATCH 44/49] MALI: valhall: upgrade DDK to g29p0-00eac0, from g28p0-00eac0 mali_csffw.bin from Valhall DDK g29(r54) is included. In addition, in kbase_device_runtime_suspend(), some lines of codes added by r54 was removed. If these lines of code are executed together with "the code for powering off the GPU with a delay on the RK platform", it will lead to a deadlock. Change-Id: I96ef4071d31f0742148cd80eba93aa0a3520daf6 Signed-off-by: Zhen Chen --- Documentation/ABI/testing/sysfs-device-mali | 133 +- .../sysfs-device-mali-coresight-source | 18 +- .../devicetree/bindings/arm/mali-valhall.txt | 257 ++ .../memory_group_manager.c | 12 + drivers/gpu/arm/valhall/Kbuild | 24 +- drivers/gpu/arm/valhall/Kconfig | 9 - drivers/gpu/arm/valhall/Makefile | 3 - .../arm/valhall/arbiter/mali_kbase_arbif.c | 261 -- .../arm/valhall/arbiter/mali_kbase_arbif.h | 15 - .../valhall/arbiter/mali_kbase_arbiter_pm.c | 71 +- drivers/gpu/arm/valhall/backend/gpu/Kbuild | 12 +- .../gpu/mali_kbase_cache_policy_backend.c | 8 +- .../gpu/mali_kbase_debug_job_fault_backend.c | 167 - .../arm/valhall/backend/gpu/mali_kbase_defs.h | 8 +- .../valhall/backend/gpu/mali_kbase_devfreq.c | 6 +- .../backend/gpu/mali_kbase_gpuprops_backend.c | 23 - .../backend/gpu/mali_kbase_instr_backend.c | 479 --- .../backend/gpu/mali_kbase_irq_linux.c | 56 +- .../valhall/backend/gpu/mali_kbase_jm_as.c | 237 -- .../valhall/backend/gpu/mali_kbase_jm_hw.c | 1280 ------- .../backend/gpu/mali_kbase_jm_internal.h | 140 - .../valhall/backend/gpu/mali_kbase_jm_rb.c | 1793 --------- .../valhall/backend/gpu/mali_kbase_jm_rb.h | 77 - .../backend/gpu/mali_kbase_js_backend.c | 371 -- .../backend/gpu/mali_kbase_js_internal.h | 72 - .../backend/gpu/mali_kbase_model_dummy.c | 486 +-- .../backend/gpu/mali_kbase_model_dummy.h | 2 - .../backend/gpu/mali_kbase_pm_always_on.c | 4 +- .../backend/gpu/mali_kbase_pm_backend.c | 311 +- .../valhall/backend/gpu/mali_kbase_pm_ca.c | 213 +- .../valhall/backend/gpu/mali_kbase_pm_ca.h | 22 +- .../backend/gpu/mali_kbase_pm_coarse_demand.c | 4 +- .../valhall/backend/gpu/mali_kbase_pm_defs.h | 76 +- .../backend/gpu/mali_kbase_pm_driver.c | 1167 +----- .../backend/gpu/mali_kbase_pm_internal.h | 191 +- .../backend/gpu/mali_kbase_pm_metrics.c | 153 +- .../backend/gpu/mali_kbase_pm_policy.c | 74 +- .../backend/gpu/mali_kbase_pm_shader_states.h | 10 +- .../arm/valhall/backend/gpu/mali_kbase_time.c | 33 + drivers/gpu/arm/valhall/build.bp | 2 - drivers/gpu/arm/valhall/context/Kbuild | 12 +- .../context/backend/mali_kbase_context_csf.c | 3 + .../context/backend/mali_kbase_context_jm.c | 266 -- .../arm/valhall/context/mali_kbase_context.c | 22 +- .../context/mali_kbase_context_internal.h | 5 +- .../ipa_control/mali_kbase_csf_ipa_control.c | 2 - .../ipa_control/mali_kbase_csf_ipa_control.h | 4 +- drivers/gpu/arm/valhall/csf/mali_kbase_csf.c | 162 +- .../valhall/csf/mali_kbase_csf_csg_debugfs.c | 6 +- .../gpu/arm/valhall/csf/mali_kbase_csf_defs.h | 45 +- .../arm/valhall/csf/mali_kbase_csf_event.c | 10 +- .../arm/valhall/csf/mali_kbase_csf_firmware.c | 137 +- .../arm/valhall/csf/mali_kbase_csf_firmware.h | 10 +- .../csf/mali_kbase_csf_firmware_no_mali.c | 103 +- .../arm/valhall/csf/mali_kbase_csf_fw_io.c | 9 +- .../arm/valhall/csf/mali_kbase_csf_fw_io.h | 52 +- .../csf/mali_kbase_csf_fw_io_no_mali.c | 8 +- .../gpu/arm/valhall/csf/mali_kbase_csf_kcpu.c | 5 +- .../gpu/arm/valhall/csf/mali_kbase_csf_kcpu.h | 2 +- .../csf/mali_kbase_csf_mcu_shared_reg.c | 6 +- .../valhall/csf/mali_kbase_csf_registers.h | 21 + .../valhall/csf/mali_kbase_csf_reset_gpu.c | 26 +- .../valhall/csf/mali_kbase_csf_scheduler.c | 195 +- .../valhall/csf/mali_kbase_csf_scheduler.h | 4 - .../csf/mali_kbase_csf_tiler_heap_reclaim.c | 80 +- .../csf/mali_kbase_csf_tiler_heap_reclaim.h | 24 +- drivers/gpu/arm/valhall/debug/Kbuild | 14 +- .../mali_kbase_debug_ktrace_codes_jm.h | 203 - .../backend/mali_kbase_debug_ktrace_defs_jm.h | 111 - .../backend/mali_kbase_debug_ktrace_jm.c | 115 - .../backend/mali_kbase_debug_ktrace_jm.h | 297 -- .../mali_kbase_debug_linux_ktrace_jm.h | 151 - .../valhall/debug/mali_kbase_debug_ktrace.h | 6 +- .../debug/mali_kbase_debug_ktrace_codes.h | 4 - .../debug/mali_kbase_debug_ktrace_defs.h | 6 +- .../debug/mali_kbase_debug_linux_ktrace.h | 4 - drivers/gpu/arm/valhall/device/Kbuild | 16 +- .../device/backend/mali_kbase_device_csf.c | 1 + .../device/backend/mali_kbase_device_hw_csf.c | 11 +- .../device/backend/mali_kbase_device_hw_jm.c | 104 - .../device/backend/mali_kbase_device_jm.c | 344 -- .../arm/valhall/device/mali_kbase_device.c | 58 +- .../arm/valhall/device/mali_kbase_device.h | 4 - .../arm/valhall/device/mali_kbase_device_hw.c | 16 +- drivers/gpu/arm/valhall/gpu/Kbuild | 9 +- .../gpu/backend/mali_kbase_gpu_fault_jm.c | 176 - drivers/gpu/arm/valhall/hw_access/Kbuild | 9 +- .../valhall/hw_access/mali_kbase_hw_access.c | 18 - .../valhall/hw_access/mali_kbase_hw_access.h | 2 - .../hw_access/mali_kbase_hw_access_regmap.h | 5 - .../mali_kbase_hw_access_regmap_legacy.h | 4 - .../regmap/mali_kbase_regmap_csf_enums.h | 4 - .../regmap/mali_kbase_regmap_csf_macros.h | 4 - .../hw_access/regmap/mali_kbase_regmap_jm.c | 3041 --------------- .../regmap/mali_kbase_regmap_jm_enums.h | 777 ---- .../regmap/mali_kbase_regmap_jm_macros.h | 308 -- .../regmap/mali_kbase_regmap_legacy_csf.h | 5 +- .../regmap/mali_kbase_regmap_legacy_jm.h | 108 - drivers/gpu/arm/valhall/hwcnt/Kbuild | 16 +- .../backend/mali_kbase_hwcnt_backend_csf.c | 1357 +++++-- .../backend/mali_kbase_hwcnt_backend_csf.h | 97 +- .../backend/mali_kbase_hwcnt_backend_csf_if.h | 20 +- .../mali_kbase_hwcnt_backend_csf_if_fw.c | 50 +- .../backend/mali_kbase_hwcnt_backend_jm.c | 981 ----- .../backend/mali_kbase_hwcnt_backend_jm.h | 58 - .../mali_kbase_hwcnt_backend_jm_watchdog.c | 841 ----- .../mali_kbase_hwcnt_backend_jm_watchdog.h | 67 - .../hwcnt/mali_kbase_hwcnt_watchdog_if.h | 15 +- .../mali_kbase_hwcnt_watchdog_if_timer.c | 22 +- drivers/gpu/arm/valhall/ipa/Kbuild | 16 +- .../mali_kbase_ipa_counter_common_jm.c | 339 -- .../mali_kbase_ipa_counter_common_jm.h | 227 -- .../ipa/backend/mali_kbase_ipa_counter_jm.c | 536 --- .../arm/valhall/ipa/mali_kbase_ipa_simple.c | 10 +- .../gpu/arm/valhall/jm/mali_kbase_jm_defs.h | 770 ---- drivers/gpu/arm/valhall/jm/mali_kbase_jm_js.h | 1002 ----- .../arm/valhall/jm/mali_kbase_jm_mem_flags.h | 53 - .../gpu/arm/valhall/jm/mali_kbase_js_defs.h | 466 --- drivers/gpu/arm/valhall/mali_csffw.bin | Bin 278528 -> 278528 bytes drivers/gpu/arm/valhall/mali_kbase.h | 231 -- drivers/gpu/arm/valhall/mali_kbase_config.c | 42 +- drivers/gpu/arm/valhall/mali_kbase_config.h | 123 +- .../arm/valhall/mali_kbase_config_defaults.h | 7 - .../gpu/arm/valhall/mali_kbase_core_linux.c | 1723 +-------- .../gpu/arm/valhall/mali_kbase_ctx_sched.c | 34 +- .../gpu/arm/valhall/mali_kbase_ctx_sched.h | 18 +- .../arm/valhall/mali_kbase_debug_job_fault.c | 544 --- .../arm/valhall/mali_kbase_debug_job_fault.h | 125 - .../arm/valhall/mali_kbase_debug_mem_zones.c | 4 +- drivers/gpu/arm/valhall/mali_kbase_defs.h | 230 +- .../gpu/arm/valhall/mali_kbase_dummy_job_wa.c | 404 -- .../gpu/arm/valhall/mali_kbase_dummy_job_wa.h | 72 - .../gpu/arm/valhall/mali_kbase_dvfs_debugfs.c | 8 +- drivers/gpu/arm/valhall/mali_kbase_event.c | 265 -- drivers/gpu/arm/valhall/mali_kbase_fence.c | 49 - drivers/gpu/arm/valhall/mali_kbase_fence.h | 88 +- .../gpu/arm/valhall/mali_kbase_fence_ops.c | 22 +- drivers/gpu/arm/valhall/mali_kbase_gpuprops.c | 13 - .../arm/valhall/mali_kbase_hwaccess_defs.h | 6 +- .../gpu/arm/valhall/mali_kbase_hwaccess_jm.h | 308 -- .../gpu/arm/valhall/mali_kbase_hwaccess_pm.h | 17 +- .../arm/valhall/mali_kbase_hwaccess_time.h | 11 +- drivers/gpu/arm/valhall/mali_kbase_io.c | 31 +- drivers/gpu/arm/valhall/mali_kbase_io.h | 4 +- .../arm/valhall/mali_kbase_ioctl_helpers.h | 29 - drivers/gpu/arm/valhall/mali_kbase_jd.c | 1492 -------- .../gpu/arm/valhall/mali_kbase_jd_debugfs.c | 189 - .../gpu/arm/valhall/mali_kbase_jd_debugfs.h | 43 - drivers/gpu/arm/valhall/mali_kbase_jm.c | 153 - drivers/gpu/arm/valhall/mali_kbase_jm.h | 117 - drivers/gpu/arm/valhall/mali_kbase_js.c | 3271 ----------------- drivers/gpu/arm/valhall/mali_kbase_js.h | 36 - .../gpu/arm/valhall/mali_kbase_js_ctx_attr.c | 324 -- .../gpu/arm/valhall/mali_kbase_js_ctx_attr.h | 152 - .../gpu/arm/valhall/mali_kbase_kinstr_jm.c | 880 ----- .../gpu/arm/valhall/mali_kbase_kinstr_jm.h | 257 -- drivers/gpu/arm/valhall/mali_kbase_mem.c | 212 +- drivers/gpu/arm/valhall/mali_kbase_mem.h | 66 +- .../gpu/arm/valhall/mali_kbase_mem_flags.h | 4 - .../gpu/arm/valhall/mali_kbase_mem_linux.c | 85 +- drivers/gpu/arm/valhall/mali_kbase_mem_pool.c | 213 +- .../arm/valhall/mali_kbase_mem_pool_debugfs.c | 8 +- .../arm/valhall/mali_kbase_mem_pool_group.c | 31 +- .../arm/valhall/mali_kbase_mem_pool_group.h | 54 +- drivers/gpu/arm/valhall/mali_kbase_pbha.c | 16 - .../gpu/arm/valhall/mali_kbase_pbha_debugfs.c | 10 - drivers/gpu/arm/valhall/mali_kbase_pm.c | 75 +- drivers/gpu/arm/valhall/mali_kbase_pm.h | 23 - .../gpu/arm/valhall/mali_kbase_reg_track.c | 128 +- .../gpu/arm/valhall/mali_kbase_reg_track.h | 38 +- drivers/gpu/arm/valhall/mali_kbase_softjobs.c | 1637 --------- drivers/gpu/arm/valhall/mali_kbase_sync.h | 119 +- .../gpu/arm/valhall/mali_kbase_sync_common.c | 12 +- .../gpu/arm/valhall/mali_kbase_sync_file.c | 271 +- drivers/gpu/arm/valhall/mali_linux_trace.h | 44 - drivers/gpu/arm/valhall/mmu/Kbuild | 11 +- .../mali_kbase_mmu_faults_decoder_luts_jm.c | 74 - .../mali_kbase_mmu_faults_decoder_luts_jm.h | 37 - .../valhall/mmu/backend/mali_kbase_mmu_jm.c | 418 --- drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.c | 409 +-- drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.h | 2 - .../mmu/mali_kbase_mmu_faults_decoder.c | 24 +- .../valhall/mmu/mali_kbase_mmu_hw_direct.c | 33 +- .../devicetree/mali_kbase_clk_rate_trace.c | 12 +- .../devicetree/mali_kbase_config_devicetree.c | 7 +- .../devicetree/mali_kbase_runtime_pm.c | 93 +- .../platform/meson/mali_kbase_config_meson.c | 7 +- .../platform/meson/mali_kbase_runtime_pm.c | 78 +- .../platform/rk/mali_kbase_config_rk.c | 2 - .../vexpress/mali_kbase_config_vexpress.c | 7 +- .../mali_kbase_config_vexpress.c | 7 +- .../mali_kbase_config_vexpress.c | 7 +- .../kernel/mali_kutf_clk_rate_trace_test.c | 6 - .../mali_kutf_irq_test_main.c | 4 - .../arm/valhall/thirdparty/mali_kbase_mmap.c | 15 - drivers/gpu/arm/valhall/tl/Kbuild | 9 +- .../tl/backend/mali_kbase_timeline_csf.c | 10 +- .../tl/backend/mali_kbase_timeline_jm.c | 86 - .../gpu/arm/valhall/tl/mali_kbase_timeline.c | 34 +- .../arm/valhall/tl/mali_kbase_timeline_io.c | 9 +- .../arm/valhall/tl/mali_kbase_timeline_priv.h | 4 - .../gpu/arm/valhall/tl/mali_kbase_tlstream.c | 10 +- .../gpu/arm/valhall/tl/mali_kbase_tlstream.h | 2 - .../arm/valhall/tl/mali_kbase_tracepoints.c | 60 +- .../arm/valhall/tl/mali_kbase_tracepoints.h | 629 +--- include/linux/memory_group_manager.h | 25 +- .../backend/gpu/mali_kbase_model_dummy.h | 4 - .../gpu/backend/mali_kbase_gpu_regmap_jm.h | 25 - .../arm/valhall/gpu/mali_kbase_gpu_regmap.h | 6 +- .../gpu/arm/valhall/jm/mali_base_jm_kernel.h | 892 ----- .../gpu/arm/valhall/jm/mali_kbase_jm_ioctl.h | 274 -- .../arm/valhall/jm/mali_kbase_jm_mem_flags.h | 65 - .../gpu/arm/valhall/mali_base_common_kernel.h | 6 - .../uapi/gpu/arm/valhall/mali_base_kernel.h | 25 +- .../uapi/gpu/arm/valhall/mali_base_mem_priv.h | 6 +- include/uapi/gpu/arm/valhall/mali_gpu_props.h | 8 + .../uapi/gpu/arm/valhall/mali_kbase_ioctl.h | 10 +- .../arm/valhall/mali_kbase_kinstr_jm_reader.h | 69 - .../gpu/arm/valhall/mali_kbase_mem_flags.h | 4 - 219 files changed, 3664 insertions(+), 36606 deletions(-) create mode 100644 Documentation/devicetree/bindings/arm/mali-valhall.txt delete mode 100644 drivers/gpu/arm/valhall/backend/gpu/mali_kbase_debug_job_fault_backend.c delete mode 100644 drivers/gpu/arm/valhall/backend/gpu/mali_kbase_instr_backend.c delete mode 100644 drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_as.c delete mode 100644 drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_hw.c delete mode 100644 drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_internal.h delete mode 100644 drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.c delete mode 100644 drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.h delete mode 100644 drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_backend.c delete mode 100644 drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_internal.h delete mode 100644 drivers/gpu/arm/valhall/context/backend/mali_kbase_context_jm.c delete mode 100644 drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_codes_jm.h delete mode 100644 drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_defs_jm.h delete mode 100644 drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.c delete mode 100644 drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.h delete mode 100644 drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_linux_ktrace_jm.h delete mode 100644 drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_jm.c delete mode 100644 drivers/gpu/arm/valhall/device/backend/mali_kbase_device_jm.c delete mode 100644 drivers/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_fault_jm.c delete mode 100644 drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm.c delete mode 100644 drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_enums.h delete mode 100644 drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_macros.h delete mode 100644 drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_jm.h delete mode 100644 drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c delete mode 100644 drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.h delete mode 100644 drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c delete mode 100644 drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.h delete mode 100644 drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.c delete mode 100644 drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.h delete mode 100644 drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_jm.c delete mode 100644 drivers/gpu/arm/valhall/jm/mali_kbase_jm_defs.h delete mode 100644 drivers/gpu/arm/valhall/jm/mali_kbase_jm_js.h delete mode 100644 drivers/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h delete mode 100644 drivers/gpu/arm/valhall/jm/mali_kbase_js_defs.h delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.c delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.h delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.c delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.h delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_event.c delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_fence.c delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_hwaccess_jm.h delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_jd.c delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.c delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.h delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_jm.c delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_jm.h delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_js.c delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_js.h delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.c delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.h delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.c delete mode 100644 drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.h delete mode 100644 drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c delete mode 100644 drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h delete mode 100644 drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_jm.c delete mode 100644 drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_jm.c delete mode 100644 include/uapi/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_regmap_jm.h delete mode 100644 include/uapi/gpu/arm/valhall/jm/mali_base_jm_kernel.h delete mode 100644 include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_ioctl.h delete mode 100644 include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h delete mode 100644 include/uapi/gpu/arm/valhall/mali_kbase_kinstr_jm_reader.h diff --git a/Documentation/ABI/testing/sysfs-device-mali b/Documentation/ABI/testing/sysfs-device-mali index 12a1667feeb2..c6326b3ca9ad 100644 --- a/Documentation/ABI/testing/sysfs-device-mali +++ b/Documentation/ABI/testing/sysfs-device-mali @@ -1,15 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software - * Foundation) and any use by you of this program is subject to the terms - * of such GNU licence. + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. * - * A copy of the licence is included with the program) and can also be obtained - * from Free Software Foundation) Inc.) 51 Franklin Street) Fifth Floor) - * Boston) MA 02110-1301) USA. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ @@ -34,16 +40,6 @@ Description: driver, On reading it provides the current DVFS sampling period, on writing a value we set the DVFS sampling period. -What: /sys/class/misc/mali%u/device/dummy_job_wa_info -Description: - This attribute is available only with platform device that - supports a Job Manager based GPU that requires a GPU workaround - to execute the dummy fragment job on all shader cores to - workaround a hang issue. - - Its a readonly attribute and on reading gives details on the - options used with the dummy workaround. - What: /sys/class/misc/mali%u/device/fw_timeout Description: This attribute is available only with mali platform @@ -78,111 +74,6 @@ Description: is supported or is powered down after suspending command stream groups. -What: /sys/class/misc/mali%u/device/js_ctx_scheduling_mode -Description: - This attribute is available only with platform device that - supports a Job Manager based GPU. This attribute is used to set - context scheduling priority for a job slot. - - On Reading it provides the currently set job slot context - priority. - - Writing 0 to this attribute sets it to the mode were - higher priority atoms will be scheduled first, regardless of - the context they belong to. Newly-runnable higher priority atoms - can preempt lower priority atoms currently running on the GPU, - even if they belong to a different context. - - Writing 1 to this attribute set it to the mode were the - highest-priority atom will be chosen from each context in turn - using a round-robin algorithm, so priority only has an effect - within the context an atom belongs to. Newly-runnable higher - priority atoms can preempt the lower priority atoms currently - running on the GPU, but only if they belong to the same context. - -What: /sys/class/misc/mali%u/device/js_scheduling_period -Description: - This attribute is available only with platform device that - supports a Job Manager based GPU. Used to set the job scheduler - tick period in nano-seconds. The Job Scheduler determines the - jobs that are run on the GPU, and for how long, Job Scheduler - makes decisions at a regular time interval determined by value - in js_scheduling_period. - -What: /sys/class/misc/mali%u/device/js_softstop_always -Description: - This attribute is available only with platform device that - supports a Job Manager based GPU. Soft-stops are disabled when - only a single context is present, this attribute is used to - enable soft-stop when only a single context is present can be - used for debug and unit-testing purposes. - -What: /sys/class/misc/mali%u/device/js_timeouts -Description: - This attribute is available only with platform device that - supports a Job Manager based GPU. It used to set the soft stop - and hard stop times for the job scheduler. - - Writing value 0 causes no change, or -1 to restore the - default timeout. - - The format used to set js_timeouts is - " - - " - - -What: /sys/class/misc/mali%u/device/lp_mem_pool_max_size -Description: - This attribute is used to set the maximum number of large pages - memory pools that the driver can contain. Large pages are of - size 2MB. On read it displays all the max size of all memory - pools and can be used to modify each individual pools as well. - -What: /sys/class/misc/mali%u/device/lp_mem_pool_size -Description: - This attribute is used to set the number of large memory pages - which should be populated, changing this value may cause - existing pages to be removed from the pool, or new pages to be - created and then added to the pool. On read it will provide - pool size for all available pools and we can modify individual - pool. - -What: /sys/class/misc/mali%u/device/mem_pool_max_size -Description: - This attribute is used to set the maximum number of small pages - for memory pools that the driver can contain. Here small pages - are of size 4KB. On read it will display the max size for all - available pools and allows us to set max size of - individual pools. - -What: /sys/class/misc/mali%u/device/mem_pool_size -Description: - This attribute is used to set the number of small memory pages - which should be populated, changing this value may cause - existing pages to be removed from the pool, or new pages to - be created and then added to the pool. On read it will provide - pool size for all available pools and we can modify individual - pool. - -What: /sys/class/misc/mali%u/device/device/mempool/ctx_default_max_size -Description: - This attribute is used to set maximum memory pool size for - all the memory pool so that the maximum amount of free memory - that each pool can hold is identical. - -What: /sys/class/misc/mali%u/device/device/mempool/lp_max_size -Description: - This attribute is used to set the maximum number of large pages - for all memory pools that the driver can contain. - Large pages are of size 2MB. - -What: /sys/class/misc/mali%u/device/device/mempool/max_size -Description: - This attribute is used to set the maximum number of small pages - for all the memory pools that the driver can contain. - Here small pages are of size 4KB. - What: /sys/class/misc/mali%u/device/pm_poweroff Description: This attribute contains the current values, represented as the diff --git a/Documentation/ABI/testing/sysfs-device-mali-coresight-source b/Documentation/ABI/testing/sysfs-device-mali-coresight-source index 58d9085b8bb6..c8453702bdd9 100644 --- a/Documentation/ABI/testing/sysfs-device-mali-coresight-source +++ b/Documentation/ABI/testing/sysfs-device-mali-coresight-source @@ -1,15 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software - * Foundation) and any use by you of this program is subject to the terms - * of such GNU licence. + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. * - * A copy of the licence is included with the program) and can also be obtained - * from Free Software Foundation) Inc.) 51 Franklin Street) Fifth Floor) - * Boston) MA 02110-1301) USA. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. * */ diff --git a/Documentation/devicetree/bindings/arm/mali-valhall.txt b/Documentation/devicetree/bindings/arm/mali-valhall.txt new file mode 100644 index 000000000000..18f6af21210d --- /dev/null +++ b/Documentation/devicetree/bindings/arm/mali-valhall.txt @@ -0,0 +1,257 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU license. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# + +* ARM Mali Midgard / Bifrost devices + + +Required properties: + +- compatible : Should be mali, replacing digits with x from the back, +until malitxx, and it must end with one of: "arm,malit6xx" or +"arm,mali-midgard" or "arm,mali-bifrost" +- reg : Physical base address of the device and length of the register area. +- interrupts : Contains the three IRQ lines required by T-6xx devices +- interrupt-names : Contains the names of IRQ resources in the order they were +provided in the interrupts property. Must contain: "JOB, "MMU", "GPU". + +Optional: + +- clocks : One or more pairs of phandle to clock and clock specifier + for the Mali device. The order is important: the first clock + shall correspond to the "clk_mali" source. Other clocks are optional + and, if present, they shall correspond to domains like "shadercores", + which is available for all GPUs, or "coregroup" and "neuralengines" + which are available for newer GPUs. Also notice that the "neuralengines" + clock domain, if present, doesn't expect a corresponding regulator. +- clock-names : Shall be set to values like: "clk_mali", "shadercores", "coregroup", + "neuralengines". Only the first one is mandatory. Most GPUs + support only the first two entries. +- mali-supply : Phandle to the top level regulator for the Mali device. + Refer to +Documentation/devicetree/bindings/regulator/regulator.txt for details. +- mem-supply : Phandle to memory regulator for the Mali device. This is optional. +- operating-points-v2 : Refer to Documentation/devicetree/bindings/power/mali-opp.txt +for details. +- quirks-gpu : Used to write to the JM_CONFIG or CSF_CONFIG register. + Should be used with care. Options passed here are used to override + certain default behavior. Note: This will override 'idvs-group-size' + field in devicetree and module param 'corestack_driver_control', + therefore if 'quirks-gpu' is used then 'idvs-group-size' and + 'corestack_driver_control' value should be incorporated into 'quirks-gpu'. +- quirks-sc : Used to write to the SHADER_CONFIG register. + Should be used with care. Options passed here are used to override + certain default behavior. +- quirks-tiler : Used to write to the TILER_CONFIG register. + Should be used with care. Options passed here are used to + disable or override certain default behavior. +- quirks-mmu : Used to write to the L2_CONFIG register. + Should be used with care. Options passed here are used to + disable or override certain default behavior. +- power-model : Sets the power model parameters. Defined power models include: + "mali-simple-power-model", "mali-g51-power-model", "mali-g52-power-model", + "mali-g52_r1-power-model", "mali-g71-power-model", "mali-g72-power-model", + "mali-g76-power-model", "mali-g77-power-model", "mali-tnax-power-model", + "mali-tbex-power-model" and "mali-tbax-power-model". + - mali-simple-power-model: this model derives the GPU power usage based + on the GPU voltage scaled by the system temperature. Note: it was + designed for the Juno platform, and may not be suitable for others. + - compatible: Should be "arm,mali-simple-power-model" + - dynamic-coefficient: Coefficient, in pW/(Hz V^2), which is + multiplied by v^2*f to calculate the dynamic power consumption. + - static-coefficient: Coefficient, in uW/V^3, which is + multiplied by v^3 to calculate the static power consumption. + - ts: An array containing coefficients for the temperature + scaling factor. This is used to scale the static power by a + factor of tsf/1000000, + where tsf = ts[3]*T^3 + ts[2]*T^2 + ts[1]*T + ts[0], + and T = temperature in degrees. + - thermal-zone: A string identifying the thermal zone used for + the GPU + - temp-poll-interval-ms: the interval at which the system + temperature is polled + - mali-g*-power-model(s): unless being stated otherwise, these models derive + the GPU power usage based on performance counters, so they are more + accurate. + - compatible: Should be, as examples, "arm,mali-g51-power-model" / + "arm,mali-g72-power-model". + - scale: the dynamic power calculated by the power model is + multiplied by a factor of 'scale'. This value should be + chosen to match a particular implementation. + - min_sample_cycles: Fall back to the simple power model if the + number of GPU cycles for a given counter dump is less than + 'min_sample_cycles'. The default value of this should suffice. + * Note: when IPA is used, two separate power models (simple and counter-based) + are used at different points so care should be taken to configure + both power models in the device tree (specifically dynamic-coefficient, + static-coefficient and scale) to best match the platform. +- power-policy : Sets the GPU power policy at probe time. Available options are + "coarse_demand" and "always_on". If not set, then "coarse_demand" is used. +- system-coherency : Sets the coherency protocol to be used for coherent + accesses made from the GPU. + If not set then no coherency is used. + - 0 : ACE-Lite + - 1 : ACE + - 31 : No coherency +- ipa-model : Sets the IPA model to be used for power management. GPU probe will fail if the + model is not found in the registered models list. If no model is specified here, + a gpu-id based model is picked if available, otherwise the default model is used. + - mali-simple-power-model: Default model used on mali +- idvs-group-size : Override the IDVS group size value. Tasks are sent to + cores in groups of N + 1, so i.e. 0xF means 16 tasks. + Valid values are between 0 to 0x3F (including). +- l2-size : Override L2 cache size on GPU that supports it. Value should be larger than the minimum + size 1KiB and smaller than the maximum size. Maximum size is Hardware integration dependent. + The value passed should be of log2(Cache Size in Bytes). + For example for a 1KiB of cache size, 0xa should be passed. +- l2-hash : Override L2 hash function on GPU that supports it +- l2-hash-values : Override L2 hash function using provided hash values, on GPUs that supports it. + It is mutually exclusive with 'l2-hash'. Only one or the other must be + used in a supported GPU. +- arbiter-if : Phandle to the arbif platform device, used to provide KBASE with an interface + to the Arbiter. This is required when using arbitration; setting to a non-NULL + value will enable arbitration. + If arbitration is in use, then there should be no external GPU control. + When arbiter-if is in use then the following must not be: + - power-model (no IPA allowed with arbitration) + - #cooling-cells + - operating-points-v2 (no dvfs in kbase with arbitration) + - system-coherency with a value of 1 (no full coherency with arbitration) +- int-id-override: list of tuples defining the IDs needed to be + set and the setting coresponding to the SYSC_ALLOC register. +- propagate-bits: Used to write to L2_CONFIG.PBHA_HWU. This bitset establishes which + PBHA bits are propagated on the AXI bus. +- mma-wa-id: Sets a GPU MMU PBHA value to be used as NONCACHEABLE. GPU Page Table Entries + (PTEs) with this PBHA value set will have uncached transactions from the GPU. + Valid values are from 1 to 15. See mgm_update_gpu_pte() in + include/linux/memory_group_manager.h for the interface to configure the PTEs. Only + supported by arch 14.8.x and later GPUs. + + +Example for a Mali GPU with 1 clock and 1 regulator: + +gpu@0xfc010000 { + compatible = "arm,malit602", "arm,malit60x", "arm,malit6xx", "arm,mali-midgard"; + reg = <0xfc010000 0x4000>; + interrupts = <0 36 4>, <0 37 4>, <0 38 4>; + interrupt-names = "JOB", "MMU", "GPU"; + + clocks = <&pclk_mali>; + clock-names = "clk_mali"; + mali-supply = <&vdd_mali>; + operating-points-v2 = <&gpu_opp_table>; + power_model@0 { + compatible = "arm,mali-simple-power-model"; + static-coefficient = <2427750>; + dynamic-coefficient = <4687>; + ts = <20000 2000 (-20) 2>; + thermal-zone = "gpu"; + }; + power_model@1 { + compatible = "arm,mali-g71-power-model"; + scale = <5>; + }; + + idvs-group-size = <0x7>; + l2-size = /bits/ 8 <0x10>; + l2-hash = /bits/ 8 <0x04>; /* or l2-hash-values = <0x12345678 0x8765 0xAB>; */ +}; + +gpu_opp_table: opp_table0 { + compatible = "operating-points-v2"; + + opp@533000000 { + opp-hz = /bits/ 64 <533000000>; + opp-microvolt = <1250000>; + }; + opp@450000000 { + opp-hz = /bits/ 64 <450000000>; + opp-microvolt = <1150000>; + }; + opp@400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <1125000>; + }; + opp@350000000 { + opp-hz = /bits/ 64 <350000000>; + opp-microvolt = <1075000>; + }; + opp@266000000 { + opp-hz = /bits/ 64 <266000000>; + opp-microvolt = <1025000>; + }; + opp@160000000 { + opp-hz = /bits/ 64 <160000000>; + opp-microvolt = <925000>; + }; + opp@100000000 { + opp-hz = /bits/ 64 <100000000>; + opp-microvolt = <912500>; + }; +}; + +Example for a Mali GPU with 2 clocks and 2 regulators: + +gpu: gpu@6e000000 { + compatible = "arm,mali-midgard"; + reg = <0x0 0x6e000000 0x0 0x200000>; + interrupts = <0 168 4>, <0 168 4>, <0 168 4>; + interrupt-names = "JOB", "MMU", "GPU"; + clocks = <&clk_mali 0>, <&clk_mali 1>; + clock-names = "clk_mali", "shadercores"; + mali-supply = <&supply0_3v3>; + mem-supply = <&supply1_3v3>; + system-coherency = <31>; + operating-points-v2 = <&gpu_opp_table>; +}; + +gpu_opp_table: opp_table0 { + compatible = "operating-points-v2", "operating-points-v2-mali"; + + opp@0 { + opp-hz = /bits/ 64 <50000000>; + opp-hz-real = /bits/ 64 <50000000>, /bits/ 64 <45000000>; + opp-microvolt = <820000>, <800000>; + opp-core-mask = /bits/ 64 <0xf>; + }; + opp@1 { + opp-hz = /bits/ 64 <40000000>; + opp-hz-real = /bits/ 64 <40000000>, /bits/ 64 <35000000>; + opp-microvolt = <720000>, <700000>; + opp-core-mask = /bits/ 64 <0x7>; + }; + opp@2 { + opp-hz = /bits/ 64 <30000000>; + opp-hz-real = /bits/ 64 <30000000>, /bits/ 64 <25000000>; + opp-microvolt = <620000>, <700000>; + opp-core-mask = /bits/ 64 <0x3>; + }; +}; + +Example for a Mali GPU supporting PBHA configuration via DTB (default): + +gpu@0xfc010000 { + ... + pbha { + int-id-override = <2 0x32>, <9 0x05>, <16 0x32>; + propagate-bits = /bits/ 8 <0x03>; + mma-wa-id = <2>; + }; + ... +}; diff --git a/drivers/base/arm/memory_group_manager/memory_group_manager.c b/drivers/base/arm/memory_group_manager/memory_group_manager.c index 6789fe87075a..2c5780d42a62 100644 --- a/drivers/base/arm/memory_group_manager/memory_group_manager.c +++ b/drivers/base/arm/memory_group_manager/memory_group_manager.c @@ -310,6 +310,18 @@ static u64 example_mgm_update_gpu_pte(struct memory_group_manager_device *const if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return pte; + /* If a page is mapped uncached on the CPU but cached on the GPU, it will be considered to + * have Mismatched Memory Attributes (MMA), and the MMA_VIOLATION bit will be set in the + * pte_flags argument. + * + * If the system requires consistent memory attributes external to the GPU, system + * integrators must allocate one of the PBHA values (range 1-15) for this feature, and + * specify the value either via the mma-wa-id devicetree property or via the mma_wa_id + * module parameter, which is then passed into this function via the pbha_id parameter. The + * GPU will continue to use cached transactions internally, but use non-cacheable + * transactions externally. Note that system integrators may choose not to set the PBHA + * value here if their system does not require it. + */ if (pte_flags & BIT(MMA_VIOLATION)) { pr_warn_once("MMA violation! Applying PBHA override workaround to PTE\n"); pte |= ((u64)pbha_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; diff --git a/drivers/gpu/arm/valhall/Kbuild b/drivers/gpu/arm/valhall/Kbuild index e2a58ea59d20..41afecc01294 100644 --- a/drivers/gpu/arm/valhall/Kbuild +++ b/drivers/gpu/arm/valhall/Kbuild @@ -63,7 +63,7 @@ endif # # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= '"g28p0-00eac0"' +MALI_RELEASE_NAME ?= '"g29p0-00eac0"' # Set up defaults if not defined by build system ifeq ($(CONFIG_MALI_VALHALL_DEBUG), y) MALI_UNIT_TEST = 1 @@ -79,13 +79,10 @@ MALI_PLATFORM_DIR := $(shell echo $(CONFIG_MALI_VALHALL_PLATFORM_NAME)) ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) MALI_JIT_PRESSURE_LIMIT_BASE = 0 - MALI_USE_CSF = 1 else MALI_JIT_PRESSURE_LIMIT_BASE ?= 1 - MALI_USE_CSF ?= 0 endif - ifneq ($(CONFIG_MALI_VALHALL_KUTF), n) MALI_KERNEL_TEST_API ?= 1 else @@ -104,7 +101,6 @@ endif # ccflags-y = \ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ - -DMALI_USE_CSF=$(MALI_USE_CSF) \ -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ -DMALI_COVERAGE=$(MALI_COVERAGE) \ @@ -112,7 +108,6 @@ ccflags-y = \ -DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \ -DMALI_PLATFORM_DIR=$(MALI_PLATFORM_DIR) - ifeq ($(KBUILD_EXTMOD),) # in-tree ccflags-y +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_VALHALL_PLATFORM_NAME) @@ -183,23 +178,6 @@ valhall_kbase-$(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) += \ mali_power_gpu_work_period_trace.o \ mali_kbase_gpu_metrics.o -ifneq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += \ - mali_kbase_jm.o \ - mali_kbase_dummy_job_wa.o \ - mali_kbase_debug_job_fault.o \ - mali_kbase_event.o \ - mali_kbase_jd.o \ - mali_kbase_jd_debugfs.o \ - mali_kbase_js.o \ - mali_kbase_js_ctx_attr.o \ - mali_kbase_kinstr_jm.o - - valhall_kbase-$(CONFIG_SYNC_FILE) += \ - mali_kbase_fence_ops.o \ - mali_kbase_fence.o -endif - INCLUDE_SUBDIR = \ $(src)/arbiter/Kbuild \ diff --git a/drivers/gpu/arm/valhall/Kconfig b/drivers/gpu/arm/valhall/Kconfig index d821acbdafe0..b54098a24b3d 100644 --- a/drivers/gpu/arm/valhall/Kconfig +++ b/drivers/gpu/arm/valhall/Kconfig @@ -324,15 +324,6 @@ config MALI_VALHALL_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE endif -config MALI_VALHALL_ARBITRATION - tristate "Enable Virtualization reference code" - depends on MALI_VALHALL - default n - help - Enables the build of several reference modules used in the reference - virtualization setup for Mali - If unsure, say N. - config MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD bool "Enable per-application GPU metrics tracepoints" depends on MALI_VALHALL diff --git a/drivers/gpu/arm/valhall/Makefile b/drivers/gpu/arm/valhall/Makefile index f5871b01a3be..1bc641fb1119 100644 --- a/drivers/gpu/arm/valhall/Makefile +++ b/drivers/gpu/arm/valhall/Makefile @@ -39,7 +39,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_VALHALL_PLATFORM_NAME ?= "devicetree" CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD ?= y CONFIG_MALI_BIFROST_GATOR_SUPPORT ?= y - CONFIG_MALI_VALHALL_ARBITRATION ?= n CONFIG_MALI_KUTF_PTM_TESTS ?= n @@ -143,7 +142,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) endif else # Prevent misuse when CONFIG_MALI_BIFROST=n - CONFIG_MALI_VALHALL_ARBITRATION = n CONFIG_MALI_VALHALL_KUTF = n CONFIG_MALI_VALHALL_KUTF_IRQ_TEST = n CONFIG_MALI_VALHALL_KUTF_CLK_RATE_TRACE = n @@ -155,7 +153,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_BIFROST \ CONFIG_MALI_VALHALL_CSF_SUPPORT \ CONFIG_MALI_BIFROST_GATOR_SUPPORT \ - CONFIG_MALI_VALHALL_ARBITRATION \ CONFIG_MALI_KUTF_PTM_TESTS \ CONFIG_MALI_VALHALL_REAL_HW \ CONFIG_MALI_BIFROST_DEVFREQ \ diff --git a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.c b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.c index 49b42a6ec2c0..b69b315ccd3d 100644 --- a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.c +++ b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.c @@ -36,264 +36,6 @@ #error "Unsupported Mali Arbiter interface version." #endif -static void on_max_config(struct device *dev, uint32_t max_l2_slices, uint32_t max_core_mask) -{ - struct kbase_device *kbdev; - - if (!dev) { - pr_err("%s(): dev is NULL", __func__); - return; - } - - kbdev = dev_get_drvdata(dev); - if (!kbdev) { - dev_err(dev, "%s(): kbdev is NULL", __func__); - return; - } - - if (!max_l2_slices || !max_core_mask) { - dev_dbg(dev, "%s(): max_config ignored as one of the fields is zero", __func__); - return; - } - - /* set the max config info in the kbase device */ - kbase_arbiter_set_max_config(kbdev, max_l2_slices, max_core_mask); -} - -/** - * on_update_freq() - Updates GPU clock frequency - * @dev: arbiter interface device handle - * @freq: GPU clock frequency value reported from arbiter - * - * call back function to update GPU clock frequency with - * new value from arbiter - */ -static void on_update_freq(struct device *dev, uint32_t freq) -{ - struct kbase_device *kbdev; - - if (!dev) { - pr_err("%s(): dev is NULL", __func__); - return; - } - - kbdev = dev_get_drvdata(dev); - if (!kbdev) { - dev_err(dev, "%s(): kbdev is NULL", __func__); - return; - } - - kbase_arbiter_pm_update_gpu_freq(&kbdev->arb.arb_freq, freq); -} - -/** - * on_gpu_stop() - sends KBASE_VM_GPU_STOP_EVT event on VM stop - * @dev: arbiter interface device handle - * - * call back function to signal a GPU STOP event from arbiter interface - */ -static void on_gpu_stop(struct device *dev) -{ - struct kbase_device *kbdev; - - if (!dev) { - pr_err("%s(): dev is NULL", __func__); - return; - } - - kbdev = dev_get_drvdata(dev); - if (!kbdev) { - dev_err(dev, "%s(): kbdev is NULL", __func__); - return; - } - - KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED(kbdev, kbdev); - KBASE_KTRACE_ADD(kbdev, ARB_GPU_STOP_REQUESTED, NULL, 0); - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT); -} - -/** - * on_gpu_granted() - sends KBASE_VM_GPU_GRANTED_EVT event on GPU granted - * @dev: arbiter interface device handle - * - * call back function to signal a GPU GRANT event from arbiter interface - */ -static void on_gpu_granted(struct device *dev) -{ - struct kbase_device *kbdev; - - if (!dev) { - pr_err("%s(): dev is NULL", __func__); - return; - } - - kbdev = dev_get_drvdata(dev); - if (!kbdev) { - dev_err(dev, "%s(): kbdev is NULL", __func__); - return; - } - - KBASE_TLSTREAM_TL_ARBITER_GRANTED(kbdev, kbdev); - KBASE_KTRACE_ADD(kbdev, ARB_GPU_GRANTED, NULL, 0); - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT); -} - -/** - * on_gpu_lost() - sends KBASE_VM_GPU_LOST_EVT event on GPU granted - * @dev: arbiter interface device handle - * - * call back function to signal a GPU LOST event from arbiter interface - */ -static void on_gpu_lost(struct device *dev) -{ - struct kbase_device *kbdev; - - if (!dev) { - pr_err("%s(): dev is NULL", __func__); - return; - } - - kbdev = dev_get_drvdata(dev); - if (!kbdev) { - dev_err(dev, "%s(): kbdev is NULL", __func__); - return; - } - KBASE_TLSTREAM_TL_ARBITER_LOST(kbdev, kbdev); - KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, 0); - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT); -} - -static int kbase_arbif_of_init(struct kbase_device *kbdev) -{ - struct arbiter_if_dev *arb_if; - struct device_node *arbiter_if_node; - struct platform_device *pdev; - - if (!IS_ENABLED(CONFIG_OF)) { - /* - * Return -ENODEV in the event CONFIG_OF is not available and let the - * internal AW check for suitability for arbitration. - */ - return -ENODEV; - } - - arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter-if", 0); - if (!arbiter_if_node) - arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter_if", 0); - if (!arbiter_if_node) { - dev_dbg(kbdev->dev, "No arbiter_if in Device Tree"); - /* no arbiter interface defined in device tree */ - kbdev->arb.arb_dev = NULL; - kbdev->arb.arb_if = NULL; - return -ENODEV; - } - - pdev = of_find_device_by_node(arbiter_if_node); - if (!pdev) { - dev_err(kbdev->dev, "Failed to find arbiter_if device"); - return -EPROBE_DEFER; - } - - if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) { - dev_err(kbdev->dev, "arbiter_if driver not available"); - put_device(&pdev->dev); - return -EPROBE_DEFER; - } - kbdev->arb.arb_dev = &pdev->dev; - arb_if = platform_get_drvdata(pdev); - if (!arb_if) { - dev_err(kbdev->dev, "arbiter_if driver not ready"); - module_put(pdev->dev.driver->owner); - put_device(&pdev->dev); - return -EPROBE_DEFER; - } - - kbdev->arb.arb_if = arb_if; - return 0; -} - -static void kbase_arbif_of_term(struct kbase_device *kbdev) -{ - if (!IS_ENABLED(CONFIG_OF)) - return; - - if (kbdev->arb.arb_dev) { - module_put(kbdev->arb.arb_dev->driver->owner); - put_device(kbdev->arb.arb_dev); - } - kbdev->arb.arb_dev = NULL; -} - - -/** - * kbase_arbif_init() - Kbase Arbiter interface initialisation. - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Initialise Kbase Arbiter interface and assign callback functions. - * - * Return: - * * 0 - the interface was initialized or was not specified - * * in the device tree. - * * -EFAULT - the interface was specified but failed to initialize. - * * -EPROBE_DEFER - module dependencies are not yet available. - */ -int kbase_arbif_init(struct kbase_device *kbdev) -{ - struct arbiter_if_arb_vm_ops ops; - struct arbiter_if_dev *arb_if; - int err = 0; - - /* Tries to init with 'arbiter-if' if present in devicetree */ - err = kbase_arbif_of_init(kbdev); - - if (err == -ENODEV) { - /* devicetree does not support arbitration */ - return -EPERM; - } - - if (err) - return err; - - ops.arb_vm_gpu_stop = on_gpu_stop; - ops.arb_vm_gpu_granted = on_gpu_granted; - ops.arb_vm_gpu_lost = on_gpu_lost; - ops.arb_vm_max_config = on_max_config; - ops.arb_vm_update_freq = on_update_freq; - - kbdev->arb.arb_freq.arb_freq = 0; - kbdev->arb.arb_freq.freq_updated = false; - mutex_init(&kbdev->arb.arb_freq.arb_freq_lock); - - arb_if = kbdev->arb.arb_if; - - if (arb_if == NULL) { - dev_err(kbdev->dev, "No arbiter interface present"); - goto failure_term; - } - - if (!arb_if->vm_ops.vm_arb_register_dev) { - dev_err(kbdev->dev, "arbiter_if registration callback not present"); - goto failure_term; - } - - /* register kbase arbiter_if callbacks */ - err = arb_if->vm_ops.vm_arb_register_dev(arb_if, kbdev->dev, &ops); - if (err) { - dev_err(kbdev->dev, "Failed to register with arbiter. (err = %d)", err); - goto failure_term; - } - - return 0; - -failure_term: - { - kbase_arbif_of_term(kbdev); - } - - if (err != -EPROBE_DEFER) - err = -EFAULT; - return err; -} /** * kbase_arbif_destroy() - De-init Kbase arbiter interface @@ -308,9 +50,6 @@ void kbase_arbif_destroy(struct kbase_device *kbdev) if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) arb_if->vm_ops.vm_arb_unregister_dev(kbdev->arb.arb_if); - { - kbase_arbif_of_term(kbdev); - } kbdev->arb.arb_if = NULL; } diff --git a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.h b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.h index c77792115e4d..8da8e3c1d0d9 100644 --- a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.h +++ b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbif.h @@ -51,21 +51,6 @@ enum kbase_arbif_evt { }; -/** - * kbase_arbif_init() - Initialize the arbiter interface functionality. - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Initialize the arbiter interface and also determines - * if Arbiter functionality is required. - * - * Return: - * * 0 - the interface was initialized or was not specified - * * in the device tree. - * * -EFAULT - the interface was specified but failed to initialize. - * * -EPROBE_DEFER - module dependencies are not yet available. - */ -int kbase_arbif_init(struct kbase_device *kbdev); - /** * kbase_arbif_destroy() - Cleanups the arbiter interface functionality. * @kbdev: The kbase device structure for the device (must be a valid pointer) diff --git a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbiter_pm.c b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbiter_pm.c index 1378e6486c37..57952106429a 100644 --- a/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbiter_pm.c +++ b/drivers/gpu/arm/valhall/arbiter/mali_kbase_arbiter_pm.c @@ -31,9 +31,7 @@ #include #include -#if MALI_USE_CSF #include -#endif /* A dmesg warning will occur if the GPU is not granted * after the following time (in milliseconds) has ellapsed. @@ -299,7 +297,9 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) arb_vm_state->vm_request_timer.function = request_timer_callback; kbdev->pm.arb_vm_state = arb_vm_state; - err = kbase_arbif_init(kbdev); + /* platform does not support arbitration */ + err = -EPERM; + if (err) { if (err != -EPERM) dev_err(kbdev->dev, "Failed to initialise arbif module. (err = %d)", err); @@ -360,10 +360,8 @@ void kbase_arbiter_pm_early_term(struct kbase_device *kbdev) cancel_request_timer(kbdev); mutex_lock(&arb_vm_state->vm_state_lock); - if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) { - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); + if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) kbase_arbif_gpu_stopped(kbdev, false); - } mutex_unlock(&arb_vm_state->vm_state_lock); destroy_workqueue(arb_vm_state->vm_arb_wq); kbase_arbif_destroy(kbdev); @@ -410,6 +408,7 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) { bool request_gpu = false; struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + unsigned long flags; lockdep_assert_held(&arb_vm_state->vm_state_lock); @@ -420,17 +419,6 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) dev_dbg(kbdev->dev, "%s %s\n", __func__, kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); - /* - * Release the interrupts on external arb_if to address Xen requirements. - * Interrupts are not released with internal arb_if as the IRQs are required - * to handle messaging to/from Arbiter/Resource Group. - */ - if (arb_vm_state->interrupts_installed - ) { - arb_vm_state->interrupts_installed = false; - kbase_release_interrupts(kbdev); - } - switch (arb_vm_state->vm_state) { case KBASE_VM_STATE_STOPPING_ACTIVE: request_gpu = true; @@ -447,7 +435,17 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) break; } - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (kbdev->pm.backend.gpu_lost_pending) { + { + kbdev->csf.firmware_reload_needed = true; + dev_warn(kbdev->dev, "%s(): GPU LOST detected, marking MCU for cold boot", + __func__); + } + } + kbase_io_set_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); + kbdev->pm.backend.gpu_lost_pending = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); { kbase_arbif_gpu_stopped(kbdev, request_gpu); @@ -550,16 +548,6 @@ static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) break; case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING); - arb_vm_state->interrupts_installed = true; - /* - * Re-install interrupts that were released for external arb_if to - * address Xen requirements. Interrupts are not released with internal - * arb_if as the IRQs are required to handle messaging to/from - * Arbiter/Resource Group. - */ - { - kbase_install_interrupts(kbdev); - } /* * GPU GRANTED received while in stop can be a result of a * repartitioning. @@ -569,7 +557,6 @@ static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) queue_work(arb_vm_state->vm_arb_wq, &arb_vm_state->vm_resume_work); break; case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); kbase_arbif_gpu_stopped(kbdev, false); kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); break; @@ -637,7 +624,6 @@ static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev) static void kbase_gpu_lost(struct kbase_device *kbdev) { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - bool handle_gpu_lost = false; lockdep_assert_held(&arb_vm_state->vm_state_lock); @@ -648,13 +634,11 @@ static void kbase_gpu_lost(struct kbase_device *kbdev) dev_warn(kbdev->dev, "GPU lost in state %s", kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); kbase_arbiter_pm_vm_gpu_stop(kbdev); - handle_gpu_lost = true; break; case KBASE_VM_STATE_STOPPING_IDLE: case KBASE_VM_STATE_STOPPING_ACTIVE: case KBASE_VM_STATE_SUSPEND_PENDING: - dev_dbg(kbdev->dev, "GPU lost while stopping"); - handle_gpu_lost = true; + dev_warn(kbdev->dev, "GPU lost while stopping"); break; case KBASE_VM_STATE_SUSPENDED: case KBASE_VM_STATE_STOPPED: @@ -668,8 +652,6 @@ static void kbase_gpu_lost(struct kbase_device *kbdev) default: break; } - if (handle_gpu_lost) - kbase_pm_handle_gpu_lost(kbdev); } /** @@ -798,6 +780,7 @@ static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev) void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, enum kbase_arbif_evt evt) { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; + unsigned long flags; if (!kbase_has_arbiter(kbdev)) return; @@ -816,6 +799,13 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, enum kbase_arbif_evt break; case KBASE_VM_GPU_LOST_EVT: dev_dbg(kbdev->dev, "KBASE_ARBIF_GPU_LOST_EVT!"); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (!kbdev->pm.backend.gpu_lost_pending) { + dev_dbg(kbdev->dev, "skipping GPU_LOST handling"); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + break; + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); kbase_gpu_lost(kbdev); break; case KBASE_VM_OS_SUSPEND_EVENT: @@ -933,10 +923,6 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; int res = 0; -#if !MALI_USE_CSF - CSTD_UNUSED(sched_lock_held); -#endif - if (!kbase_has_arbiter(kbdev)) return res; @@ -949,7 +935,8 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED_GPU_REQUESTED); kbase_arbif_gpu_request(kbdev); start_request_timer(kbdev); - } else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_ACTIVE) { + } else if (suspend_handler != KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE && + arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_ACTIVE) { res = 1; break; } else if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) @@ -985,15 +972,11 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, atomic_inc(&kbdev->pm.gpu_users_waiting); mutex_unlock(&arb_vm_state->vm_state_lock); kbase_pm_unlock(kbdev); -#if MALI_USE_CSF if (sched_lock_held) kbase_csf_scheduler_unlock(kbdev); -#endif kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); -#if MALI_USE_CSF if (sched_lock_held) kbase_csf_scheduler_lock(kbdev); -#endif kbase_pm_lock(kbdev); mutex_lock(&arb_vm_state->vm_state_lock); atomic_dec(&kbdev->pm.gpu_users_waiting); diff --git a/drivers/gpu/arm/valhall/backend/gpu/Kbuild b/drivers/gpu/arm/valhall/backend/gpu/Kbuild index 78f362d9046e..dd6b3bb038c8 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/Kbuild +++ b/drivers/gpu/arm/valhall/backend/gpu/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -33,16 +33,6 @@ valhall_kbase-y += \ backend/gpu/mali_kbase_l2_mmu_config.o \ backend/gpu/mali_kbase_clk_rate_trace_mgr.o -ifeq ($(MALI_USE_CSF),0) - valhall_kbase-y += \ - backend/gpu/mali_kbase_instr_backend.o \ - backend/gpu/mali_kbase_jm_as.o \ - backend/gpu/mali_kbase_debug_job_fault_backend.o \ - backend/gpu/mali_kbase_jm_hw.o \ - backend/gpu/mali_kbase_jm_rb.o \ - backend/gpu/mali_kbase_js_backend.o -endif - valhall_kbase-$(CONFIG_MALI_VALHALL_DEVFREQ) += \ backend/gpu/mali_kbase_devfreq.o diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_cache_policy_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_cache_policy_backend.c index aa8436420234..ed42d5349978 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_cache_policy_backend.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_cache_policy_backend.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,7 +27,6 @@ void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, u32 mode) kbdev->current_gpu_coherency_mode = mode; -#if MALI_USE_CSF if (kbdev->gpu_props.gpu_id.arch_id >= GPU_ID_ARCH_MAKE(12, 0, 1)) { /* AMBA_ENABLE present from 12.0.1 */ u32 val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(AMBA_ENABLE)); @@ -38,14 +37,10 @@ void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, u32 mode) /* Fallback to COHERENCY_ENABLE for older versions */ kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(COHERENCY_ENABLE), mode); } -#else /* MALI_USE_CSF */ - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(COHERENCY_ENABLE), mode); -#endif /* MALI_USE_CSF */ } void kbase_amba_set_shareable_cache_support(struct kbase_device *kbdev) { -#if MALI_USE_CSF /* AMBA registers only present from 12.0.1 */ if (kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(12, 0, 1)) @@ -60,5 +55,4 @@ void kbase_amba_set_shareable_cache_support(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(AMBA_ENABLE), val); } } -#endif /* MALI_USE_CSF */ } diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_debug_job_fault_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_debug_job_fault_backend.c deleted file mode 100644 index af8d1e3af87c..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_debug_job_fault_backend.c +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include -#include "mali_kbase_debug_job_fault.h" - -#if IS_ENABLED(CONFIG_DEBUG_FS) - -/*GPU_CONTROL_REG(r)*/ -static unsigned int gpu_control_reg_snapshot[] = { GPU_CONTROL_ENUM(GPU_ID), - GPU_CONTROL_ENUM(SHADER_READY), - GPU_CONTROL_ENUM(TILER_READY), - GPU_CONTROL_ENUM(L2_READY) }; - -/* JOB_CONTROL_REG(r) */ -static unsigned int job_control_reg_snapshot[] = { JOB_CONTROL_ENUM(JOB_IRQ_MASK), - JOB_CONTROL_ENUM(JOB_IRQ_STATUS) }; - -/* JOB_SLOT_REG(n,r) */ -static unsigned int job_slot_reg_snapshot[] = { - JOB_SLOT_ENUM(0, HEAD) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, TAIL) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, AFFINITY) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, CONFIG) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, STATUS) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, HEAD_NEXT) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, AFFINITY_NEXT) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, CONFIG_NEXT) - JOB_SLOT_BASE_ENUM(0) -}; - -/*MMU_CONTROL_REG(r)*/ -static unsigned int mmu_reg_snapshot[] = { MMU_CONTROL_ENUM(IRQ_MASK), - MMU_CONTROL_ENUM(IRQ_STATUS) }; - -/* MMU_AS_REG(n,r) */ -static unsigned int as_reg_snapshot[] = { MMU_AS_ENUM(0, TRANSTAB) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, TRANSCFG) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, MEMATTR) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, FAULTSTATUS) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, FAULTADDRESS) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, STATUS) - MMU_AS_BASE_ENUM(0) }; - -bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, int reg_range) -{ - uint i, j; - int offset = 0; - uint slot_number; - uint as_number; - - if (kctx->reg_dump == NULL) - return false; - - slot_number = kctx->kbdev->gpu_props.num_job_slots; - as_number = kctx->kbdev->gpu_props.num_address_spaces; - - /* get the GPU control registers*/ - for (i = 0; i < ARRAY_SIZE(gpu_control_reg_snapshot); i++) { - kctx->reg_dump[offset] = gpu_control_reg_snapshot[i]; - if (kbase_reg_is_size64(kctx->kbdev, kctx->reg_dump[offset])) - offset += 4; - else - offset += 2; - } - - /* get the Job control registers*/ - for (i = 0; i < ARRAY_SIZE(job_control_reg_snapshot); i++) { - kctx->reg_dump[offset] = job_control_reg_snapshot[i]; - if (kbase_reg_is_size64(kctx->kbdev, kctx->reg_dump[offset])) - offset += 4; - else - offset += 2; - } - - /* get the Job Slot registers*/ - for (j = 0; j < slot_number; j++) { - for (i = 0; i < ARRAY_SIZE(job_slot_reg_snapshot); i++) { - kctx->reg_dump[offset] = JOB_SLOT_BASE_OFFSET(j) + job_slot_reg_snapshot[i]; - if (kbase_reg_is_size64(kctx->kbdev, kctx->reg_dump[offset])) - offset += 4; - else - offset += 2; - } - } - - /* get the MMU registers*/ - for (i = 0; i < ARRAY_SIZE(mmu_reg_snapshot); i++) { - kctx->reg_dump[offset] = mmu_reg_snapshot[i]; - if (kbase_reg_is_size64(kctx->kbdev, kctx->reg_dump[offset])) - offset += 4; - else - offset += 2; - } - - /* get the Address space registers*/ - for (j = 0; j < as_number; j++) { - for (i = 0; i < ARRAY_SIZE(as_reg_snapshot); i++) { - kctx->reg_dump[offset] = MMU_AS_BASE_OFFSET(j) + as_reg_snapshot[i]; - if (kbase_reg_is_size64(kctx->kbdev, kctx->reg_dump[offset])) - offset += 4; - else - offset += 2; - } - } - - WARN_ON(offset >= (reg_range * 2 / 4)); - - /* set the termination flag*/ - kctx->reg_dump[offset] = REGISTER_DUMP_TERMINATION_FLAG; - kctx->reg_dump[offset + 1] = REGISTER_DUMP_TERMINATION_FLAG; - - dev_dbg(kctx->kbdev->dev, "kbase_job_fault_reg_snapshot_init:%d\n", offset); - - return true; -} - -bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx) -{ - int offset = 0; - u32 reg_enum; - u64 val64; - - if (kctx->reg_dump == NULL) - return false; - - while (kctx->reg_dump[offset] != REGISTER_DUMP_TERMINATION_FLAG) { - reg_enum = kctx->reg_dump[offset]; - /* Get register offset from enum */ - kbase_reg_get_offset(kctx->kbdev, reg_enum, &kctx->reg_dump[offset]); - - if (kbase_reg_is_size64(kctx->kbdev, reg_enum)) { - val64 = kbase_reg_read64(kctx->kbdev, reg_enum); - - /* offset computed offset to get _HI offset */ - kctx->reg_dump[offset + 2] = kctx->reg_dump[offset] + 4; - - kctx->reg_dump[offset + 1] = (u32)(val64 & 0xFFFFFFFF); - kctx->reg_dump[offset + 3] = (u32)(val64 >> 32); - offset += 4; - } else { - kctx->reg_dump[offset + 1] = kbase_reg_read32(kctx->kbdev, reg_enum); - offset += 2; - } - } - return true; -} - -#endif diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_defs.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_defs.h index bff0b7235736..fa745bf6b471 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_defs.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -98,12 +98,6 @@ struct slot_rb { * The hwaccess_lock (a spinlock) must be held when accessing this structure */ struct kbase_backend_data { -#if !MALI_USE_CSF - struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS]; - struct hrtimer scheduling_timer; - - bool timer_running; -#endif bool suspend_timer; atomic_t reset_gpu; diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_devfreq.c index 22be72239fe3..db6a475a5301 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_devfreq.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_devfreq.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -195,7 +195,7 @@ static int kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *s stat->current_frequency = kbdev->current_nominal_freq; stat->private_data = NULL; -#if MALI_USE_CSF && defined CONFIG_DEVFREQ_THERMAL +#if defined CONFIG_DEVFREQ_THERMAL if (!kbdev->devfreq_profile.is_cooling_device) kbase_ipa_reset_data(kbdev); #endif @@ -399,7 +399,6 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) opp_freq); continue; } -#if MALI_USE_CSF if (kbase_csf_dev_has_ne(kbdev)) { u64 neural_present = kbdev->gpu_props.neural_present; u64 sc_with_ne = shader_present & neural_present; @@ -423,7 +422,6 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) continue; } } -#endif /* MALI_USE_CSF */ core_count_p = of_get_property(node, "opp-core-count", NULL); if (core_count_p) { diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_gpuprops_backend.c index 1cd3dafcb452..cdca4d3ed6b8 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_gpuprops_backend.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_gpuprops_backend.c @@ -44,22 +44,10 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(STACK_PRESENT))) regdump->stack_present = kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(STACK_PRESENT)); -#if !MALI_USE_CSF - regdump->js_present = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(JS_PRESENT)); - /* Not a valid register on TMIX */ - - /* TGOx specific register */ - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_THREAD_TLS_ALLOC)) - regdump->thread_tls_alloc = - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_TLS_ALLOC)); -#endif /* !MALI_USE_CSF */ - regdump->thread_max_threads = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_MAX_THREADS)); if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(THREAD_MAX_WORKGROUP_SIZE))) regdump->thread_max_workgroup_size = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_MAX_WORKGROUP_SIZE)); -#if MALI_USE_CSF -#endif /* MALI_USE_CSF */ if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(THREAD_MAX_BARRIER_SIZE))) regdump->thread_max_barrier_size = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_MAX_BARRIER_SIZE)); @@ -73,7 +61,6 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CORE_FEATURES)) regdump->core_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(CORE_FEATURES)); -#if MALI_USE_CSF if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(GPU_FEATURES))) regdump->gpu_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_FEATURES)); /* Only applicable to GPUs with power control domain registers */ @@ -82,19 +69,11 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop regdump->neural_present = kbase_reg_read64(kbdev, HOST_POWER_ENUM(NEURAL_PRESENT)); } -#endif /* MALI_USE_CSF */ regdump->tiler_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(TILER_FEATURES)); regdump->l2_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(L2_FEATURES)); regdump->mem_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(MEM_FEATURES)); regdump->mmu_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(MMU_FEATURES)); -#if !MALI_USE_CSF - for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) - regdump->js_features[i] = kbase_reg_read32(kbdev, GPU_JS_FEATURES_OFFSET(i)); -#endif /* !MALI_USE_CSF */ - -#if MALI_USE_CSF -#endif /* MALI_USE_CSF */ { for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) regdump->texture_features[i] = @@ -131,14 +110,12 @@ int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, regdump->l2_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(L2_FEATURES)); regdump->l2_config = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG)); -#if MALI_USE_CSF if (kbase_hw_has_l2_slice_hash_feature(kbdev)) { uint i; for (i = 0; i < GPU_L2_SLICE_HASH_COUNT; i++) regdump->l2_slice_hash[i] = kbase_reg_read32(kbdev, GPU_L2_SLICE_HASH_OFFSET(i)); } -#endif /* MALI_USE_CSF */ if (!kbase_io_has_gpu(kbdev)) return -EIO; diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_instr_backend.c deleted file mode 100644 index 62bb507c64d1..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_instr_backend.c +++ /dev/null @@ -1,479 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * GPU backend instrumentation APIs. - */ - -#include -#include -#include -#include -#include -#include - -#define WAIT_FOR_DUMP_TIMEOUT_MS 5000 - -static int wait_prfcnt_ready(struct kbase_device *kbdev) -{ - u32 val; - const u32 timeout_us = - kbase_get_timeout_ms(kbdev, KBASE_PRFCNT_ACTIVE_TIMEOUT) * USEC_PER_MSEC; - const int err = kbase_reg_poll32_timeout(kbdev, GPU_CONTROL_ENUM(GPU_STATUS), val, - !(val & GPU_STATUS_PRFCNT_ACTIVE), 0, timeout_us, - false); - if (err) { - dev_err(kbdev->dev, "PRFCNT_ACTIVE bit stuck\n"); - return -EBUSY; - } - return 0; -} - -int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_instr_hwcnt_enable *enable) -{ - unsigned long flags; - int err = -EINVAL; - u32 irq_mask; - u32 prfcnt_config; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* alignment failure */ - if ((enable->dump_buffer == 0ULL) || (enable->dump_buffer & (2048 - 1))) - return err; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) { - /* Instrumentation is already enabled */ - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - return err; - } - - if (!kbase_io_has_gpu(kbdev)) { - /* GPU has been removed by Arbiter */ - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - return err; - } - - /* Enable interrupt */ - irq_mask = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), - irq_mask | PRFCNT_SAMPLE_COMPLETED); - - /* In use, this context is the owner */ - kbdev->hwcnt.kctx = kctx; - /* Remember the dump address so we can reprogram it later */ - kbdev->hwcnt.addr = enable->dump_buffer; - kbdev->hwcnt.addr_bytes = enable->dump_buffer_bytes; - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - /* Configure */ - prfcnt_config = (u32)kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; -#ifdef CONFIG_MALI_VALHALL_PRFCNT_SET_SELECT_VIA_DEBUG_FS - prfcnt_config |= (u32)kbdev->hwcnt.backend.override_counter_set - << PRFCNT_CONFIG_SETSELECT_SHIFT; -#else - prfcnt_config |= (u32)enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT; -#endif - - /* Wait until prfcnt config register can be written */ - err = wait_prfcnt_ready(kbdev); - if (err) - return err; - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_CONFIG), - prfcnt_config | PRFCNT_CONFIG_MODE_OFF); - - /* Wait until prfcnt is disabled before writing configuration registers */ - err = wait_prfcnt_ready(kbdev); - if (err) - return err; - - kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(PRFCNT_BASE), enable->dump_buffer); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_JM_EN), enable->fe_bm); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_SHADER_EN), enable->shader_bm); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_MMU_L2_EN), enable->mmu_l2_bm); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_TILER_EN), enable->tiler_bm); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_CONFIG), - prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL); - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - dev_dbg(kbdev->dev, "HW counters dumping set-up for context %pK", kctx); - return 0; -} - -static void kbasep_instr_hwc_disable_hw_prfcnt(struct kbase_device *kbdev) -{ - u32 irq_mask; - - lockdep_assert_held(&kbdev->hwaccess_lock); - lockdep_assert_held(&kbdev->hwcnt.lock); - - if (!kbase_io_has_gpu(kbdev)) - /* GPU has been removed by Arbiter */ - return; - - /* Disable interrupt */ - irq_mask = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), - irq_mask & ~PRFCNT_SAMPLE_COMPLETED); - - /* Wait until prfcnt config register can be written, then disable the counters. - * Return value is ignored as we are disabling anyway. - */ - wait_prfcnt_ready(kbdev); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(PRFCNT_CONFIG), 0); - - kbdev->hwcnt.kctx = NULL; - kbdev->hwcnt.addr = 0ULL; - kbdev->hwcnt.addr_bytes = 0ULL; -} - -int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) -{ - unsigned long flags, pm_flags; - struct kbase_device *kbdev = kctx->kbdev; - const unsigned long timeout = msecs_to_jiffies(WAIT_FOR_DUMP_TIMEOUT_MS); - unsigned int remaining; - - while (1) { - spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) { - /* Instrumentation is in unrecoverable error state, - * there is nothing for us to do. - */ - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - /* Already disabled, return no error. */ - return 0; - } - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DISABLED) { - /* Instrumentation is not enabled */ - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - return -EINVAL; - } - - if (kbdev->hwcnt.kctx != kctx) { - /* Instrumentation has been setup for another context */ - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - return -EINVAL; - } - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) - break; - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - - /* Ongoing dump/setup - wait for its completion */ - remaining = wait_event_timeout(kbdev->hwcnt.backend.wait, - kbdev->hwcnt.backend.triggered != 0, timeout); - - if (remaining == 0) - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_UNRECOVERABLE_ERROR; - } - - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; - kbdev->hwcnt.backend.triggered = 0; - - kbasep_instr_hwc_disable_hw_prfcnt(kbdev); - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - - dev_dbg(kbdev->dev, "HW counters dumping disabled for context %pK", kctx); - - return 0; -} - -int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) -{ - unsigned long flags; - int err = -EINVAL; - struct kbase_device *kbdev = kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - if (kbdev->hwcnt.kctx != kctx) { - /* The instrumentation has been setup for another context */ - goto unlock; - } - - if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { - /* HW counters are disabled or another dump is ongoing, or we're - * resetting, or we are in unrecoverable error state. - */ - goto unlock; - } - - if (!kbase_io_has_gpu(kbdev)) { - /* GPU has been removed by Arbiter */ - goto unlock; - } - - kbdev->hwcnt.backend.triggered = 0; - - /* Mark that we're dumping - the PF handler can signal that we faulted - */ - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; - - /* Wait until prfcnt is ready to request dump */ - err = wait_prfcnt_ready(kbdev); - if (err) - goto unlock; - - /* Reconfigure the dump address */ - kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(PRFCNT_BASE), kbdev->hwcnt.addr); - - /* Start dumping */ - KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, kbdev->hwcnt.addr); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_PRFCNT_SAMPLE); - - dev_dbg(kbdev->dev, "HW counters dumping done for context %pK", kctx); - -unlock: - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - return err; -} -KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); - -bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, bool *const success) -{ - unsigned long flags; - bool complete = false; - struct kbase_device *kbdev = kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE) { - *success = true; - complete = true; - } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { - *success = false; - complete = true; - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; - } - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - return complete; -} -KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); - -void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - /* If the state is in unrecoverable error, we already wake_up the waiter - * and don't need to do any action when sample is done. - */ - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) { - /* All finished and idle */ - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - } - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -} - -int kbase_instr_hwcnt_wait_for_dump(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - unsigned long flags; - int err; - - unsigned long remaining; - const unsigned long timeout = msecs_to_jiffies(WAIT_FOR_DUMP_TIMEOUT_MS); - - /* Wait for dump & cache clean to complete */ - remaining = wait_event_timeout(kbdev->hwcnt.backend.wait, - kbdev->hwcnt.backend.triggered != 0, timeout); - if (remaining == 0) { - err = -ETIME; - /* Set the backend state so it's clear things have gone bad (could be a HW issue) - */ - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_UNRECOVERABLE_ERROR; - goto timed_out; - } - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { - err = -EINVAL; - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; - } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) { - err = -EIO; - } else { - /* Dump done */ - KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_IDLE); - err = 0; - } - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -timed_out: - return err; -} - -int kbase_instr_hwcnt_clear(struct kbase_context *kctx) -{ - unsigned long flags; - int err = -EINVAL; - struct kbase_device *kbdev = kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - /* Check it's the context previously set up and we're not in IDLE - * state. - */ - if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) - goto unlock; - - if (!kbase_io_has_gpu(kbdev)) { - /* GPU has been removed by Arbiter */ - goto unlock; - } - - /* Wait until prfcnt is ready to clear */ - err = wait_prfcnt_ready(kbdev); - if (err) - goto unlock; - - /* Clear the counters */ - KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_PRFCNT_CLEAR); - -unlock: - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - return err; -} -KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear); - -void kbase_instr_hwcnt_on_unrecoverable_error(struct kbase_device *kbdev) -{ - unsigned long flags; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - /* If we already in unrecoverable error state, early return. */ - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) { - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - return; - } - - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_UNRECOVERABLE_ERROR; - - /* Need to disable HW if it's not disabled yet. */ - if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_DISABLED) - kbasep_instr_hwc_disable_hw_prfcnt(kbdev); - - /* Wake up any waiters. */ - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -} -KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_on_unrecoverable_error); - -void kbase_instr_hwcnt_on_before_reset(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - /* A reset is the only way to exit the unrecoverable error state */ - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_UNRECOVERABLE_ERROR) - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -} -KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_on_before_reset); - -int kbase_instr_backend_init(struct kbase_device *kbdev) -{ - spin_lock_init(&kbdev->hwcnt.lock); - - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; - - init_waitqueue_head(&kbdev->hwcnt.backend.wait); - -#ifdef CONFIG_MALI_VALHALL_PRFCNT_SET_SELECT_VIA_DEBUG_FS -/* Use the build time option for the override default. */ -#if defined(CONFIG_MALI_VALHALL_PRFCNT_SET_SECONDARY) - kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_SECONDARY; -#elif defined(CONFIG_MALI_VALHALL_PRFCNT_SET_TERTIARY) - kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_TERTIARY; -#else - /* Default to primary */ - kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_PHYSICAL_SET_PRIMARY; -#endif -#endif - return 0; -} - -void kbase_instr_backend_term(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} - -#ifdef CONFIG_MALI_VALHALL_PRFCNT_SET_SELECT_VIA_DEBUG_FS -void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev) -{ - /* No validation is done on the debugfs input. Invalid input could cause - * performance counter errors. This is acceptable since this is a debug - * only feature and users should know what they are doing. - * - * Valid inputs are the values accepted bythe SET_SELECT bits of the - * PRFCNT_CONFIG register as defined in the architecture specification. - */ - debugfs_create_u8("hwcnt_set_select", 0644, kbdev->mali_debugfs_directory, - (u8 *)&kbdev->hwcnt.backend.override_counter_set); -} -#endif diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_irq_linux.c index 11bb1c57ca96..05290d403ad5 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_irq_linux.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_irq_linux.c @@ -62,12 +62,8 @@ static irqreturn_t kbase_job_irq_handler(int irq, void *data) dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); -#if MALI_USE_CSF /* call the csf interrupt handler */ kbase_csf_interrupt(kbdev, val); -#else - kbase_job_done(kbdev, val); -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -108,7 +104,6 @@ static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) return IRQ_HANDLED; } -#if MALI_USE_CSF static irqreturn_t kbase_pwr_irq_handler(int irq, void *data) { unsigned long flags; @@ -138,7 +133,6 @@ static irqreturn_t kbase_pwr_irq_handler(int irq, void *data) return irq_state; } -#endif /* MALI_USE_CSF */ static irqreturn_t kbase_gpuonly_irq_handler(int irq, void *data) @@ -181,7 +175,6 @@ static irqreturn_t kbase_gpuonly_irq_handler(int irq, void *data) static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) { irqreturn_t irq_state = kbase_gpuonly_irq_handler(irq, data); -#if MALI_USE_CSF struct kbase_device *kbdev = kbase_untag(data); /* Skip if HOST_POWER page is not available */ @@ -189,7 +182,6 @@ static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) if (kbase_pwr_irq_handler(irq, data) == IRQ_HANDLED) irq_state = IRQ_HANDLED; } -#endif /* MALI_USE_CSF */ return irq_state; } @@ -350,6 +342,40 @@ static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) return HRTIMER_NORESTART; } +struct interrupt_masks { + u32 job; + u32 mmu; + u32 gpu; + u32 pwr; +}; + +static void store_interrupt_masks(struct interrupt_masks *masks, struct kbase_device *const kbdev) +{ + /* Store all interrupt masks */ + masks->job = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK)); + masks->mmu = kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK)); + masks->gpu = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)); + if (kbdev->pm.backend.has_host_pwr_iface) + masks->pwr = kbase_reg_read32(kbdev, HOST_POWER_ENUM(PWR_IRQ_MASK)); + + /* Set all masks to 0 to disable all interrupt sources */ + kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK), 0x0); + kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), 0x0); + kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), 0x0); + if (kbdev->pm.backend.has_host_pwr_iface) + kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_MASK), 0x0); +} + +static void restore_interrupt_masks(struct interrupt_masks *masks, struct kbase_device *const kbdev) +{ + /* Restore all interrupt masks to their previous values */ + kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK), masks->job); + kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), masks->mmu); + kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), masks->gpu); + if (kbdev->pm.backend.has_host_pwr_iface) + kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_MASK), masks->pwr); +} + /** * validate_interrupt - Validate an interrupt * @kbdev: Kbase device @@ -364,9 +390,9 @@ static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) static int validate_interrupt(struct kbase_device *const kbdev, u32 tag) { int err = 0; + struct interrupt_masks masks = { 0 }; irq_handler_t handler; const int irq = (kbdev->nr_irqs == 1) ? 0 : tag; - u32 old_mask_val; u16 mask_offset; u16 rawstat_offset; @@ -389,10 +415,8 @@ static int validate_interrupt(struct kbase_device *const kbdev, u32 tag) return -EINVAL; } - /* store old mask */ - old_mask_val = kbase_reg_read32(kbdev, mask_offset); - /* mask interrupts */ - kbase_reg_write32(kbdev, mask_offset, 0x0); + /* store masks and disable all possible interrupt sources */ + store_interrupt_masks(&masks, kbdev); if (kbdev->irqs[irq].irq) { /* release original handler and install test handler */ @@ -441,8 +465,8 @@ static int validate_interrupt(struct kbase_device *const kbdev, u32 tag) err = -EINVAL; } } - /* restore old mask */ - kbase_reg_write32(kbdev, mask_offset, old_mask_val); + /* restore old masks */ + restore_interrupt_masks(&masks, kbdev); return err; } @@ -487,7 +511,6 @@ int kbase_install_interrupts(struct kbase_device *kbdev) { u32 irq_index; -#if MALI_USE_CSF if (kbdev->gpu_props.gpu_id.arch_id >= GPU_ID_ARCH_MAKE(14, 8, 0)) { if (kbdev->nr_irqs != 1) { dev_err(kbdev->dev, "Incorrect number of irq entries (%u)", kbdev->nr_irqs); @@ -499,7 +522,6 @@ int kbase_install_interrupts(struct kbase_device *kbdev) return -EINVAL; } } -#endif /* MALI_USE_CSF */ for (irq_index = 0; irq_index < kbdev->nr_irqs; irq_index++) { const int result = request_irq(kbdev->irqs[irq_index].irq, kbase_get_interrupt_handler(kbdev, irq_index), diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_as.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_as.c deleted file mode 100644 index 960ac9905343..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_as.c +++ /dev/null @@ -1,237 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Register backend context / address space management - */ - -#include -#include -#include - -/** - * assign_and_activate_kctx_addr_space - Assign an AS to a context - * @kbdev: Kbase device - * @kctx: Kbase context - * @current_as: Address Space to assign - * - * Assign an Address Space (AS) to a context, and add the context to the Policy. - * - * This includes - * setting up the global runpool_irq structure and the context on the AS, - * Activating the MMU on the AS, - * Allowing jobs to be submitted on the AS. - * - * Context: - * kbasep_js_kctx_info.jsctx_mutex held, - * kbasep_js_device_data.runpool_mutex held, - * AS transaction mutex held, - * Runpool IRQ lock held - */ -static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, - struct kbase_context *kctx, - struct kbase_as *current_as) -{ - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - - CSTD_UNUSED(current_as); - - lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); - lockdep_assert_held(&js_devdata->runpool_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - -#if !MALI_USE_CSF - /* Attribute handling */ - kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); -#endif - - /* Allow it to run jobs */ - kbasep_js_set_submit_allowed(js_devdata, kctx); - - kbase_js_runpool_inc_context_count(kbdev, kctx); -} - -bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js) -{ - int i; - - if (kbdev->hwaccess.active_kctx[js] == kctx) { - /* Context is already active */ - return true; - } - - for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { - if (kbdev->as_to_kctx[i] == kctx) { - /* Context already has ASID - mark as active */ - return true; - } - } - - /* Context does not have address space assigned */ - return false; -} - -void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - int as_nr = kctx->as_nr; - - if (as_nr == KBASEP_AS_NR_INVALID) { - WARN(1, "Attempting to release context without ASID\n"); - return; - } - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (atomic_read(&kctx->refcount) != 1) { - WARN(1, "Attempting to release active ASID\n"); - return; - } - - kbasep_js_clear_submit_allowed(&kbdev->js_data, kctx); - - kbase_ctx_sched_release_ctx(kctx); - kbase_js_runpool_dec_context_count(kbdev, kctx); -} - -void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - CSTD_UNUSED(kbdev); - CSTD_UNUSED(kctx); -} - -int kbase_backend_find_and_release_free_address_space(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - unsigned long flags; - int i; - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { - struct kbasep_js_kctx_info *as_js_kctx_info; - struct kbase_context *as_kctx; - - as_kctx = kbdev->as_to_kctx[i]; - as_js_kctx_info = &as_kctx->jctx.sched_info; - - /* Don't release privileged or active contexts, or contexts with - * jobs running. - * Note that a context will have at least 1 reference (which - * was previously taken by kbasep_js_schedule_ctx()) until - * descheduled. - */ - if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) && - atomic_read(&as_kctx->refcount) == 1) { - if (!kbase_ctx_sched_inc_refcount_nolock(as_kctx)) { - WARN(1, "Failed to retain active context\n"); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return KBASEP_AS_NR_INVALID; - } - - kbasep_js_clear_submit_allowed(js_devdata, as_kctx); - - /* Drop and retake locks to take the jsctx_mutex on the - * context we're about to release without violating lock - * ordering - */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - /* Release context from address space */ - mutex_lock(&as_js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - - kbasep_js_runpool_release_ctx_nolock(kbdev, as_kctx); - - if (!kbase_ctx_flag(as_kctx, KCTX_SCHEDULED)) { - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, as_kctx, true); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); - - return i; - } - - /* Context was retained while locks were dropped, - * continue looking for free AS - */ - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - } - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return KBASEP_AS_NR_INVALID; -} - -bool kbase_backend_use_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, int as_nr) -{ - struct kbasep_js_device_data *js_devdata; - struct kbase_as *new_address_space = NULL; - int js; - - js_devdata = &kbdev->js_data; - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - if (kbdev->hwaccess.active_kctx[js] == kctx) { - WARN(1, "Context is already scheduled in\n"); - return false; - } - } - - new_address_space = &kbdev->as[as_nr]; - - lockdep_assert_held(&js_devdata->runpool_mutex); - lockdep_assert_held(&kbdev->mmu_hw_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - - assign_and_activate_kctx_addr_space(kbdev, kctx, new_address_space); - - if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) { - /* We need to retain it to keep the corresponding address space - */ - kbase_ctx_sched_retain_ctx_refcount(kctx); - } - - return true; -} diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_hw.c deleted file mode 100644 index d815fd80ebb9..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_hw.c +++ /dev/null @@ -1,1280 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Base kernel job manager APIs - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev); -static u64 kbasep_apply_limited_core_mask(const struct kbase_device *kbdev, const u64 affinity, - const u64 limited_core_mask); - -static u64 kbase_job_write_affinity(struct kbase_device *kbdev, base_jd_core_req core_req, - unsigned int js, const u64 limited_core_mask) -{ - u64 affinity; - bool skip_affinity_check = false; - - if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) == BASE_JD_REQ_T) { - /* Tiler-only atom, affinity value can be programed as 0 */ - affinity = 0; - skip_affinity_check = true; - } else if ((core_req & - (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) { - affinity = kbdev->pm.backend.shaders_avail & kbdev->pm.debug_core_mask[js]; - - /* Bifrost onwards GPUs only have 1 coherent group which is equal to - * shader_present - */ - affinity &= kbdev->gpu_props.curr_config.shader_present; - } else { - /* Use all cores */ - affinity = kbdev->pm.backend.shaders_avail & kbdev->pm.debug_core_mask[js]; - } - - if (core_req & BASE_JD_REQ_LIMITED_CORE_MASK) { - /* Limiting affinity due to BASE_JD_REQ_LIMITED_CORE_MASK by applying the limited core mask. */ - affinity = kbasep_apply_limited_core_mask(kbdev, affinity, limited_core_mask); - } - - if (unlikely(!affinity && !skip_affinity_check)) { -#ifdef CONFIG_MALI_VALHALL_DEBUG - u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); - - WARN_ON(!(shaders_ready & kbdev->pm.backend.shaders_avail)); -#endif - - affinity = kbdev->pm.backend.shaders_avail; - - if (core_req & BASE_JD_REQ_LIMITED_CORE_MASK) { - /* Limiting affinity again to make sure it only enables shader cores with backed TLS memory. */ - affinity = - kbasep_apply_limited_core_mask(kbdev, affinity, limited_core_mask); - -#ifdef CONFIG_MALI_VALHALL_DEBUG - /* affinity should never be 0 */ - WARN_ON(!affinity); -#endif - } - } - - kbase_reg_write64(kbdev, JOB_SLOT_OFFSET(js, AFFINITY_NEXT), affinity); - - return affinity; -} - -static inline bool kbasep_jm_wait_js_free(struct kbase_device *kbdev, unsigned int js, - struct kbase_context *kctx) -{ - u32 val; - const u32 timeout_us = kbdev->js_data.js_free_wait_time_ms * USEC_PER_MSEC; - /* wait for the JS_COMMAND_NEXT register to reach the given status value */ - const int err = kbase_reg_poll32_timeout(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), val, - !val, 0, timeout_us, false); - - if (!err) - return true; - - dev_err(kbdev->dev, "Timeout in waiting for job slot %u to become free for ctx %d_%u", js, - kctx->tgid, kctx->id); - - return false; -} - -int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, unsigned int js) -{ - struct kbase_context *kctx; - u32 cfg; - u64 jc_head = katom->jc; - u64 affinity; - struct slot_rb *ptr_slot_rb = &kbdev->hwaccess.backend.slot_rb[js]; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - kctx = katom->kctx; - - /* Command register must be available */ - if (!kbasep_jm_wait_js_free(kbdev, js, kctx)) - return -EPERM; - - dev_dbg(kctx->kbdev->dev, "Write JS_HEAD_NEXT 0x%llx for atom %pK\n", jc_head, - (void *)katom); - - kbase_reg_write64(kbdev, JOB_SLOT_OFFSET(js, HEAD_NEXT), jc_head); - - affinity = kbase_job_write_affinity(kbdev, katom->core_req, js, kctx->limited_core_mask); - - /* start MMU, medium priority, cache clean/flush on end, clean/flush on - * start - */ - cfg = (u32)kctx->as_nr; - - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION) && - !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) - cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; - - if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) { - /* Force a cache maintenance operation if the newly submitted - * katom to the slot is from a different kctx. For a JM GPU - * that has the feature KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, - * applies a FLUSH_INV_SHADER_OTHER. Otherwise, do a - * FLUSH_CLEAN_INVALIDATE. - */ - u64 tagged_kctx = ptr_slot_rb->last_kctx_tagged; - - if (tagged_kctx != SLOT_RB_NULL_TAG_VAL && tagged_kctx != SLOT_RB_TAG_KCTX(kctx)) { - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER)) - cfg |= JS_CONFIG_START_FLUSH_INV_SHADER_OTHER; - else - cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; - } else - cfg |= JS_CONFIG_START_FLUSH_NO_ACTION; - } else - cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; - - if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && - !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) - cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; - else if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE)) - cfg |= JS_CONFIG_END_FLUSH_CLEAN; - else - cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; - - cfg |= JS_CONFIG_THREAD_PRI(8); - - if (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED) - cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; - - if (!ptr_slot_rb->job_chain_flag) { - cfg |= JS_CONFIG_JOB_CHAIN_FLAG; - katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN; - ptr_slot_rb->job_chain_flag = true; - } else { - katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN; - ptr_slot_rb->job_chain_flag = false; - } - - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, CONFIG_NEXT), cfg); - - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION)) - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, FLUSH_ID_NEXT), katom->flush_id); - - /* Write an approximate start timestamp. - * It's approximate because there might be a job in the HEAD register. - */ - katom->start_timestamp = ktime_get_raw(); - - /* GO ! */ - dev_dbg(kbdev->dev, "JS: Submitting atom %pK from ctx %pK to js[%d] with head=0x%llx", - katom, kctx, js, jc_head); - - KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, (u32)affinity); - - KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, kctx, js, kbase_jd_atom_id(kctx, katom), - TL_JS_EVENT_START); - - KBASE_TLSTREAM_TL_ATTRIB_ATOM_CONFIG(kbdev, katom, jc_head, affinity, cfg); - KBASE_TLSTREAM_TL_RET_CTX_LPU(kbdev, kctx, &kbdev->gpu_props.js_features[katom->slot_nr]); - KBASE_TLSTREAM_TL_RET_ATOM_AS(kbdev, katom, &kbdev->as[kctx->as_nr]); - KBASE_TLSTREAM_TL_RET_ATOM_LPU(kbdev, katom, &kbdev->gpu_props.js_features[js], - "ctx_nr,atom_nr"); - kbase_kinstr_jm_atom_hw_submit(katom); - - /* Update the slot's last katom submission kctx */ - ptr_slot_rb->last_kctx_tagged = SLOT_RB_TAG_KCTX(kctx); - - trace_sysgraph_gpu(SGR_SUBMIT, kctx->id, kbase_jd_atom_id(kctx, katom), js); - - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), JS_COMMAND_START); - - return 0; -} - -/** - * kbasep_job_slot_update_head_start_timestamp - Update timestamp - * @kbdev: kbase device - * @js: job slot - * @end_timestamp: timestamp - * - * Update the start_timestamp of the job currently in the HEAD, based on the - * fact that we got an IRQ for the previous set of completed jobs. - * - * The estimate also takes into account the time the job was submitted, to - * work out the best estimate (which might still result in an over-estimate to - * the calculated time spent) - */ -static void kbasep_job_slot_update_head_start_timestamp(struct kbase_device *kbdev, unsigned int js, - ktime_t end_timestamp) -{ - ktime_t timestamp_diff; - struct kbase_jd_atom *katom; - - /* Checking the HEAD position for the job slot */ - katom = kbase_gpu_inspect(kbdev, js, 0); - if (katom != NULL) { - timestamp_diff = ktime_sub(end_timestamp, katom->start_timestamp); - if (ktime_to_ns(timestamp_diff) >= 0) { - /* Only update the timestamp if it's a better estimate - * than what's currently stored. This is because our - * estimate that accounts for the throttle time may be - * too much of an overestimate - */ - katom->start_timestamp = end_timestamp; - } - } -} - -/** - * kbasep_trace_tl_event_lpu_softstop - Call event_lpu_softstop timeline - * tracepoint - * @kbdev: kbase device - * @js: job slot - * - * Make a tracepoint call to the instrumentation module informing that - * softstop happened on given lpu (job slot). - */ -static void kbasep_trace_tl_event_lpu_softstop(struct kbase_device *kbdev, unsigned int js) -{ - KBASE_TLSTREAM_TL_EVENT_LPU_SOFTSTOP(kbdev, &kbdev->gpu_props.js_features[js]); -} - -void kbase_job_done(struct kbase_device *kbdev, u32 done) -{ - u32 count = 0; - ktime_t end_timestamp; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ, NULL, NULL, 0, done); - - end_timestamp = ktime_get_raw(); - - while (done) { - unsigned int i; - u32 failed = done >> 16; - - /* treat failed slots as finished slots */ - u32 finished = (done & 0xFFFF) | failed; - - /* Note: This is inherently unfair, as we always check for lower - * numbered interrupts before the higher numbered ones. - */ - i = (unsigned int)ffs((int)finished) - 1u; - - do { - u32 nr_done; - u32 active; - u32 completion_code = BASE_JD_EVENT_DONE; /* assume OK */ - u64 job_tail = 0; - - if (failed & (1u << i)) { - /* read out the job slot status code if the job - * slot reported failure - */ - completion_code = - kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(i, STATUS)); - - if (completion_code == BASE_JD_EVENT_STOPPED) { - u64 job_head; - - KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, NULL, i, 0, - TL_JS_EVENT_SOFT_STOP); - - kbasep_trace_tl_event_lpu_softstop(kbdev, i); - - /* Soft-stopped job - read the value of - * JS_TAIL so that the job chain can - * be resumed - */ - job_tail = - kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(i, TAIL)); - job_head = - kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(i, HEAD)); - /* For a soft-stopped job chain js_tail should - * same as the js_head, but if not then the - * job chain was incorrectly marked as - * soft-stopped. In such case we should not - * be resuming the job chain from js_tail and - * report the completion_code as UNKNOWN. - */ - if (job_tail != job_head) - completion_code = BASE_JD_EVENT_UNKNOWN; - - } else if (completion_code == BASE_JD_EVENT_NOT_STARTED) { - /* PRLAM-10673 can cause a TERMINATED - * job to come back as NOT_STARTED, - * but the error interrupt helps us - * detect it - */ - completion_code = BASE_JD_EVENT_TERMINATED; - } - - kbase_gpu_irq_evict(kbdev, i, completion_code); - - /* Some jobs that encounter a BUS FAULT may - * result in corrupted state causing future - * jobs to hang. Reset GPU before allowing - * any other jobs on the slot to continue. - */ - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_3076)) { - if (completion_code == BASE_JD_EVENT_JOB_BUS_FAULT) { - if (kbase_prepare_to_reset_gpu_locked( - kbdev, RESET_FLAGS_NONE)) - kbase_reset_gpu_locked(kbdev); - } - } - } - - kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_CLEAR), - done & ((1u << i) | (1u << (i + 16)))); - active = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_JS_STATE)); - - if (((active >> i) & 1) == 0 && (((done >> (i + 16)) & 1) == 0)) { - /* There is a potential race we must work - * around: - * - * 1. A job slot has a job in both current and - * next registers - * 2. The job in current completes - * successfully, the IRQ handler reads - * RAWSTAT and calls this function with the - * relevant bit set in "done" - * 3. The job in the next registers becomes the - * current job on the GPU - * 4. Sometime before the JOB_IRQ_CLEAR line - * above the job on the GPU _fails_ - * 5. The IRQ_CLEAR clears the done bit but not - * the failed bit. This atomically sets - * JOB_IRQ_JS_STATE. However since both jobs - * have now completed the relevant bits for - * the slot are set to 0. - * - * If we now did nothing then we'd incorrectly - * assume that _both_ jobs had completed - * successfully (since we haven't yet observed - * the fail bit being set in RAWSTAT). - * - * So at this point if there are no active jobs - * left we check to see if RAWSTAT has a failure - * bit set for the job slot. If it does we know - * that there has been a new failure that we - * didn't previously know about, so we make sure - * that we record this in active (but we wait - * for the next loop to deal with it). - * - * If we were handling a job failure (i.e. done - * has the relevant high bit set) then we know - * that the value read back from - * JOB_IRQ_JS_STATE is the correct number of - * remaining jobs because the failed job will - * have prevented any futher jobs from starting - * execution. - */ - u32 rawstat = - kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_RAWSTAT)); - - if ((rawstat >> (i + 16)) & 1) { - /* There is a failed job that we've - * missed - add it back to active - */ - active |= (1u << i); - } - } - - dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", completion_code); - - nr_done = kbase_backend_nr_atoms_submitted(kbdev, i); - nr_done -= (active >> i) & 1; - nr_done -= (active >> (i + 16)) & 1; - - if (nr_done == 0 || nr_done > SLOT_RB_SIZE) { - dev_warn(kbdev->dev, "Spurious interrupt on slot %u", i); - - goto spurious; - } - - count += nr_done; - - while (nr_done) { - if (likely(nr_done == 1)) { - kbase_gpu_complete_hw(kbdev, i, completion_code, job_tail, - &end_timestamp); - kbase_jm_try_kick_all(kbdev); - } else { - /* More than one job has completed. - * Since this is not the last job being - * reported this time it must have - * passed. This is because the hardware - * will not allow further jobs in a job - * slot to complete until the failed job - * is cleared from the IRQ status. - */ - kbase_gpu_complete_hw(kbdev, i, BASE_JD_EVENT_DONE, 0, - &end_timestamp); -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - /* Increment the end timestamp value by 1 ns to - * avoid having the same value for 'start_time_ns' - * and 'end_time_ns' for the 2nd atom whose job - * completion IRQ got merged with the 1st atom. - */ - end_timestamp = ktime_add(end_timestamp, ns_to_ktime(1)); -#endif - } - nr_done--; - } -spurious: - done = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_RAWSTAT)); - - failed = done >> 16; - finished = (done & 0xFFFF) | failed; - if (done) - end_timestamp = ktime_get_raw(); - } while (finished & (1u << i)); - - kbasep_job_slot_update_head_start_timestamp(kbdev, i, end_timestamp); - } - - if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_COMMITTED) { - /* If we're trying to reset the GPU then we might be able to do - * it early (without waiting for a timeout) because some jobs - * have completed - */ - kbasep_try_reset_gpu_early_locked(kbdev); - } - KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ_END, NULL, NULL, 0, count); -} - -void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, unsigned int js, - u32 action, base_jd_core_req core_reqs, - struct kbase_jd_atom *target_katom) -{ -#if KBASE_KTRACE_ENABLE - u32 status_reg_before; - u64 job_in_head_before; - u32 status_reg_after; - - WARN_ON(action & (~(u32)JS_COMMAND_MASK)); - - /* Check the head pointer */ - job_in_head_before = kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(js, HEAD)); - status_reg_before = kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(js, STATUS)); -#endif - - if (action == JS_COMMAND_SOFT_STOP) { - if (kbase_jd_katom_is_protected(target_katom)) { -#ifdef CONFIG_MALI_VALHALL_DEBUG - dev_dbg(kbdev->dev, - "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%x", - (unsigned int)core_reqs); -#else - CSTD_UNUSED(core_reqs); -#endif /* CONFIG_MALI_VALHALL_DEBUG */ - return; - } - - /* We are about to issue a soft stop, so mark the atom as having - * been soft stopped - */ - target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED; - - /* Mark the point where we issue the soft-stop command */ - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(kbdev, target_katom); - - action = (target_katom->atom_flags & KBASE_KATOM_FLAGS_JOBCHAIN) ? - JS_COMMAND_SOFT_STOP_1 : - JS_COMMAND_SOFT_STOP_0; - } else if (action == JS_COMMAND_HARD_STOP) { - target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED; - - action = (target_katom->atom_flags & KBASE_KATOM_FLAGS_JOBCHAIN) ? - JS_COMMAND_HARD_STOP_1 : - JS_COMMAND_HARD_STOP_0; - } - - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, COMMAND), action); - -#if KBASE_KTRACE_ENABLE - status_reg_after = kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(js, STATUS)); - if (status_reg_after == BASE_JD_EVENT_ACTIVE) { - struct kbase_jd_atom *head; - struct kbase_context *head_kctx; - - head = kbase_gpu_inspect(kbdev, js, 0); - if (unlikely(!head)) { - dev_err(kbdev->dev, "Can't get a katom from js(%d)\n", js); - return; - } - head_kctx = head->kctx; - - if (status_reg_before == BASE_JD_EVENT_ACTIVE) - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, head, - job_in_head_before, js); - else - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js); - - switch (action) { - case JS_COMMAND_SOFT_STOP: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, head_kctx, head, head->jc, js); - break; - case JS_COMMAND_SOFT_STOP_0: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, head, head->jc, - js); - break; - case JS_COMMAND_SOFT_STOP_1: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, head, head->jc, - js); - break; - case JS_COMMAND_HARD_STOP: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, head_kctx, head, head->jc, js); - break; - case JS_COMMAND_HARD_STOP_0: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, head, head->jc, - js); - break; - case JS_COMMAND_HARD_STOP_1: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, - js); - break; - default: - WARN(1, "Unknown action %d on atom %pK in kctx %pK\n", action, - (void *)target_katom, (void *)target_katom->kctx); - break; - } - } else { - if (status_reg_before == BASE_JD_EVENT_ACTIVE) - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, - job_in_head_before, js); - else - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js); - - switch (action) { - case JS_COMMAND_SOFT_STOP: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, js); - break; - case JS_COMMAND_SOFT_STOP_0: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, 0, js); - break; - case JS_COMMAND_SOFT_STOP_1: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, 0, js); - break; - case JS_COMMAND_HARD_STOP: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, js); - break; - case JS_COMMAND_HARD_STOP_0: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, 0, js); - break; - case JS_COMMAND_HARD_STOP_1: - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js); - break; - default: - WARN(1, "Unknown action %d on atom %pK in kctx %pK\n", action, - (void *)target_katom, (void *)target_katom->kctx); - break; - } - } -#endif -} - -void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - unsigned int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) - kbase_job_slot_hardstop(kctx, i, NULL); -} - -void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, - struct kbase_jd_atom *target_katom) -{ - struct kbase_device *kbdev; - unsigned int target_js = target_katom->slot_nr; - int i; - bool stop_sent = false; - - kbdev = kctx->kbdev; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, target_js); i++) { - struct kbase_jd_atom *slot_katom; - - slot_katom = kbase_gpu_inspect(kbdev, target_js, i); - if (!slot_katom) - continue; - - if (kbase_js_atom_runs_before(kbdev, target_katom, slot_katom, - KBASE_ATOM_ORDERING_FLAG_SEQNR)) { - if (!stop_sent) - KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED(kbdev, target_katom); - - kbase_job_slot_softstop(kbdev, target_js, slot_katom); - stop_sent = true; - } - } -} - -void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - unsigned long timeout = msecs_to_jiffies(ZAP_TIMEOUT); - - timeout = wait_event_timeout(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0, - (long)timeout); - - if (timeout != 0) - timeout = wait_event_timeout(kctx->jctx.sched_info.ctx.is_scheduled_wait, - !kbase_ctx_flag(kctx, KCTX_SCHEDULED), (long)timeout); - - /* Neither wait timed out; all done! */ - if (timeout != 0) - goto exit; - - if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) { - dev_err(kbdev->dev, - "Issuing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", - ZAP_TIMEOUT); - kbase_reset_gpu(kbdev); - } - - /* Wait for the reset to complete */ - kbase_reset_gpu_wait(kbdev); -exit: - dev_dbg(kbdev->dev, "Zap: Finished Context %pK", kctx); - - /* Ensure that the signallers of the waitqs have finished */ - mutex_lock(&kctx->jctx.lock); - mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - mutex_unlock(&kctx->jctx.lock); -} - -u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) -{ - u32 flush_id = 0; - - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION)) { - mutex_lock(&kbdev->pm.lock); - if (kbase_io_is_gpu_powered(kbdev)) - flush_id = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(LATEST_FLUSH)); - mutex_unlock(&kbdev->pm.lock); - } - - return flush_id; -} - -int kbase_job_slot_init(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - return 0; -} -KBASE_EXPORT_TEST_API(kbase_job_slot_init); - -void kbase_job_slot_halt(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} - -void kbase_job_slot_term(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} -KBASE_EXPORT_TEST_API(kbase_job_slot_term); - -/** - * kbase_job_slot_softstop_swflags - Soft-stop a job with flags - * @kbdev: The kbase device - * @js: The job slot to soft-stop - * @target_katom: The job that should be soft-stopped (or NULL for any job) - * @sw_flags: Flags to pass in about the soft-stop - * - * Context: - * The job slot lock must be held when calling this function. - * The job slot must not already be in the process of being soft-stopped. - * - * Soft-stop the specified job slot, with extra information about the stop - * - * Where possible any job in the next register is evicted before the soft-stop. - */ -void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, unsigned int js, - struct kbase_jd_atom *target_katom, u32 sw_flags) -{ - dev_dbg(kbdev->dev, "Soft-stop atom %pK with flags 0x%x (s:%d)\n", target_katom, sw_flags, - js); - - if (sw_flags & JS_COMMAND_MASK) { - WARN(true, "Atom %pK in kctx %pK received non-NOP flags %d\n", (void *)target_katom, - target_katom ? (void *)target_katom->kctx : NULL, sw_flags); - sw_flags &= ~((u32)JS_COMMAND_MASK); - } - kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom, - JS_COMMAND_SOFT_STOP | sw_flags); -} - -void kbase_job_slot_softstop(struct kbase_device *kbdev, unsigned int js, - struct kbase_jd_atom *target_katom) -{ - kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u); -} - -void kbase_job_slot_hardstop(struct kbase_context *kctx, unsigned int js, - struct kbase_jd_atom *target_katom) -{ - struct kbase_device *kbdev = kctx->kbdev; - bool stopped; - - stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js, target_katom, - JS_COMMAND_HARD_STOP); - CSTD_UNUSED(stopped); -} - -void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, - base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom) -{ - u32 hw_action = action & JS_COMMAND_MASK; - - CSTD_UNUSED(core_reqs); - - /* For soft-stop, don't enter if soft-stop not allowed, or isn't - * causing disjoint. - */ - if (hw_action == JS_COMMAND_SOFT_STOP && (kbase_jd_katom_is_protected(target_katom) || - (0 == (action & JS_COMMAND_SW_CAUSES_DISJOINT)))) - return; - - /* Nothing to do if already logged disjoint state on this atom */ - if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) - return; - - target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT; - kbase_disjoint_state_up(kbdev); -} - -void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, struct kbase_jd_atom *target_katom) -{ - if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) { - target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT; - kbase_disjoint_state_down(kbdev); - } -} - -int kbase_reset_gpu_prevent_and_wait(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - WARN(true, "%s Not implemented for JM GPUs", __func__); - return -EINVAL; -} - -int kbase_reset_gpu_try_prevent(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - WARN(true, "%s Not implemented for JM GPUs", __func__); - return -EINVAL; -} - -void kbase_reset_gpu_allow(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - WARN(true, "%s Not implemented for JM GPUs", __func__); -} - -void kbase_reset_gpu_assert_prevented(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - WARN(true, "%s Not implemented for JM GPUs", __func__); -} - -void kbase_reset_gpu_assert_failed_or_prevented(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - WARN(true, "%s Not implemented for JM GPUs", __func__); -} - -static void kbase_debug_dump_registers(struct kbase_device *kbdev) -{ - unsigned int i; - - kbase_io_history_dump(kbdev); - - dev_err(kbdev->dev, "Register state:"); - dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS))); - dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x", - kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_RAWSTAT)), - kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_JS_STATE))); - for (i = 0; i < 3; i++) { - dev_err(kbdev->dev, " JS%u_STATUS=0x%08x JS%u_HEAD=0x%016llx", i, - kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(i, STATUS)), i, - kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(i, HEAD))); - } - dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", - kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_RAWSTAT)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_FAULTSTATUS))); - dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)), - kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK)), - kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK))); - dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(PWR_OVERRIDE0)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(PWR_OVERRIDE1))); - dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(SHADER_CONFIG)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_MMU_CONFIG))); - dev_err(kbdev->dev, " TILER_CONFIG=0x%08x JM_CONFIG=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(TILER_CONFIG)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(JM_CONFIG))); -} - -static void kbasep_reset_timeout_worker(struct work_struct *data) -{ - unsigned long flags; - struct kbase_device *kbdev; - ktime_t end_timestamp = ktime_get_raw(); - struct kbasep_js_device_data *js_devdata; - bool silent = false; - - kbdev = container_of(data, struct kbase_device, hwaccess.backend.reset_work); - - js_devdata = &kbdev->js_data; - - if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_SILENT) - silent = true; - - if (kbase_is_quick_reset_enabled(kbdev)) - silent = true; - - KBASE_KTRACE_ADD_JM(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0); - - /* Disable GPU hardware counters. - * This call will block until counters are disabled. - */ - kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); - - /* Make sure the timer has completed - this cannot be done from - * interrupt context, so this cannot be done within - * kbasep_try_reset_gpu_early. - */ - hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); - - if (kbase_pm_context_active_handle_suspend(kbdev, - KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { - /* This would re-activate the GPU. Since it's already idle, - * there's no need to reset it - */ - atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); - kbase_disjoint_state_down(kbdev); - wake_up(&kbdev->hwaccess.backend.reset_wait); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return; - } - - WARN(kbdev->irq_reset_flush, "%s: GPU reset already in flight\n", __func__); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - spin_lock(&kbdev->mmu_mask_change); - kbase_pm_reset_start_locked(kbdev); - - /* We're about to flush out the IRQs and their bottom half's */ - kbdev->irq_reset_flush = true; - - /* Disable IRQ to avoid IRQ handlers to kick in after releasing the - * spinlock; this also clears any outstanding interrupts - */ - kbase_pm_disable_interrupts_nolock(kbdev); - - spin_unlock(&kbdev->mmu_mask_change); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* Ensure that any IRQ handlers have finished - * Must be done without any locks IRQ handlers will take - */ - kbase_synchronize_irqs(kbdev); - - /* Flush out any in-flight work items */ - kbase_flush_mmu_wqs(kbdev); - - /* The flush has completed so reset the active indicator */ - kbdev->irq_reset_flush = false; - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TMIX_8463)) { - u64 val; - const u32 timeout_us = - kbase_get_timeout_ms(kbdev, KBASE_CLEAN_CACHE_TIMEOUT) * USEC_PER_MSEC; - /* Ensure that L2 is not transitioning when we send the reset command */ - const int err = kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(L2_PWRTRANS), val, - !val, 0, timeout_us, false); - - WARN(err, "L2 power transition timed out while trying to reset\n"); - } - - mutex_lock(&kbdev->pm.lock); - /* We hold the pm lock, so there ought to be a current policy */ - if (unlikely(!kbdev->pm.backend.pm_current_policy)) - dev_warn(kbdev->dev, "No power policy set!"); - - /* All slot have been soft-stopped and we've waited - * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we - * assume that anything that is still left on the GPU is stuck there and - * we'll kill it when we reset the GPU - */ - - if (!silent) - dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", RESET_TIMEOUT); - - /* Output the state of some interesting registers to help in the - * debugging of GPU resets - */ - if (!silent) - kbase_debug_dump_registers(kbdev); - - /* Complete any jobs that were still on the GPU */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbdev->protected_mode = false; - if (!kbdev->pm.backend.protected_entry_transition_override) - kbase_backend_reset(kbdev, &end_timestamp); - kbase_pm_metrics_update(kbdev, NULL); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* Tell hardware counters a reset is about to occur. - * If the instr backend is in an unrecoverable error state (e.g. due to - * HW being unresponsive), this will transition the backend out of - * it, on the assumption a reset will fix whatever problem there was. - */ - kbase_instr_hwcnt_on_before_reset(kbdev); - - /* Reset the GPU */ - kbase_pm_init_hw(kbdev, 0); - - mutex_unlock(&kbdev->pm.lock); - - mutex_lock(&js_devdata->runpool_mutex); - - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_ctx_sched_restore_all_as(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - - kbase_pm_enable_interrupts(kbdev); - - kbase_disjoint_state_down(kbdev); - - mutex_unlock(&js_devdata->runpool_mutex); - - mutex_lock(&kbdev->pm.lock); - - kbase_pm_reset_complete(kbdev); - - /* Find out what cores are required now */ - kbase_pm_update_cores_state(kbdev); - - /* Synchronously request and wait for those cores, because if - * instrumentation is enabled it would need them immediately. - */ - kbase_pm_wait_for_desired_state(kbdev); - - mutex_unlock(&kbdev->pm.lock); - - atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); - - wake_up(&kbdev->hwaccess.backend.reset_wait); - if (!silent) - dev_err(kbdev->dev, "Reset complete"); - - /* Try submitting some jobs to restart processing */ - KBASE_KTRACE_ADD_JM(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0); - kbase_js_sched_all(kbdev); - - /* Process any pending slot updates */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_backend_slot_update(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - kbase_pm_context_idle(kbdev); - - /* Re-enable GPU hardware counters */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - KBASE_KTRACE_ADD_JM(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0); -} - -static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) -{ - struct kbase_device *kbdev = - container_of(timer, struct kbase_device, hwaccess.backend.reset_timer); - - /* Reset still pending? */ - if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_COMMITTED, - KBASE_RESET_GPU_HAPPENING) == KBASE_RESET_GPU_COMMITTED) - queue_work(kbdev->hwaccess.backend.reset_workq, - &kbdev->hwaccess.backend.reset_work); - - return HRTIMER_NORESTART; -} - -/* - * If all jobs are evicted from the GPU then we can reset the GPU - * immediately instead of waiting for the timeout to elapse - */ - -static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) -{ - unsigned int i; - u32 pending_jobs = 0; - - /* Count the number of jobs */ - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) - pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i); - - if (pending_jobs > 0) { - /* There are still jobs on the GPU - wait */ - return; - } - - /* To prevent getting incorrect registers when dumping failed job, - * skip early reset. - */ - if (atomic_read(&kbdev->job_fault_debug) > 0) - return; - - /* Check that the reset has been committed to (i.e. kbase_reset_gpu has - * been called), and that no other thread beat this thread to starting - * the reset - */ - if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_COMMITTED, - KBASE_RESET_GPU_HAPPENING) != KBASE_RESET_GPU_COMMITTED) { - /* Reset has already occurred */ - return; - } - - queue_work(kbdev->hwaccess.backend.reset_workq, &kbdev->hwaccess.backend.reset_work); -} - -static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbasep_try_reset_gpu_early_locked(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -/** - * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU - * @kbdev: kbase device - * @flags: Bitfield indicating impact of reset (see flag defines) - * - * This function soft-stops all the slots to ensure that as many jobs as - * possible are saved. - * - * Return: boolean which should be interpreted as follows: - * true - Prepared for reset, kbase_reset_gpu_locked should be called. - * false - Another thread is performing a reset, kbase_reset_gpu should - * not be called. - */ -bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev, unsigned int flags) -{ - unsigned int i; - - if (kbase_io_is_aw_removed(kbdev)) { - /* GPU access has been removed, reset will be done by - * Arbiter instead - */ - return false; - } - - if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR) - kbase_instr_hwcnt_on_unrecoverable_error(kbdev); - - if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING, - KBASE_RESET_GPU_PREPARED) != KBASE_RESET_GPU_NOT_PENDING) { - /* Some other thread is already resetting the GPU */ - return false; - } - - kbase_disjoint_state_up(kbdev); - - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) - kbase_job_slot_softstop(kbdev, i, NULL); - - return true; -} - -bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev, unsigned int flags) -{ - unsigned long lock_flags; - bool ret; - - spin_lock_irqsave(&kbdev->hwaccess_lock, lock_flags); - ret = kbase_prepare_to_reset_gpu_locked(kbdev, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, lock_flags); - - return ret; -} -KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); - -/* - * This function should be called after kbase_prepare_to_reset_gpu if it - * returns true. It should never be called without a corresponding call to - * kbase_prepare_to_reset_gpu. - * - * After this function is called (or not called if kbase_prepare_to_reset_gpu - * returned false), the caller should wait for - * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset - * has completed. - */ -void kbase_reset_gpu(struct kbase_device *kbdev) -{ - /* Note this is an assert/atomic_set because it is a software issue for - * a race to be occurring here - */ - if (WARN_ON(atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_PREPARED)) - return; - atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_COMMITTED); - - if (!kbase_is_quick_reset_enabled(kbdev)) - dev_err(kbdev->dev, - "Preparing to soft-reset GPU: Waiting (up to %d ms) for all jobs to complete soft-stop\n", - kbdev->reset_timeout_ms); - - hrtimer_start(&kbdev->hwaccess.backend.reset_timer, - HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), HRTIMER_MODE_REL); - - /* Try resetting early */ - kbasep_try_reset_gpu_early(kbdev); -} -KBASE_EXPORT_TEST_API(kbase_reset_gpu); - -void kbase_reset_gpu_locked(struct kbase_device *kbdev) -{ - /* Note this is an assert/atomic_set because it is a software issue for - * a race to be occurring here - */ - if (WARN_ON(atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_PREPARED)) - return; - atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_COMMITTED); - - if (!kbase_is_quick_reset_enabled(kbdev)) - dev_err(kbdev->dev, - "Preparing to soft-reset GPU: Waiting (up to %d ms) for all jobs to complete soft-stop\n", - kbdev->reset_timeout_ms); - hrtimer_start(&kbdev->hwaccess.backend.reset_timer, - HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms), HRTIMER_MODE_REL); - - /* Try resetting early */ - kbasep_try_reset_gpu_early_locked(kbdev); -} - -int kbase_reset_gpu_silent(struct kbase_device *kbdev) -{ - if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING, - KBASE_RESET_GPU_SILENT) != KBASE_RESET_GPU_NOT_PENDING) { - /* Some other thread is already resetting the GPU */ - return -EAGAIN; - } - - kbase_disjoint_state_up(kbdev); - - queue_work(kbdev->hwaccess.backend.reset_workq, &kbdev->hwaccess.backend.reset_work); - - return 0; -} - -bool kbase_reset_gpu_is_active(struct kbase_device *kbdev) -{ - if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_NOT_PENDING) - return false; - - return true; -} - -bool kbase_reset_gpu_is_not_pending(struct kbase_device *kbdev) -{ - return atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_NOT_PENDING; -} - -int kbase_reset_gpu_wait(struct kbase_device *kbdev) -{ - wait_event(kbdev->hwaccess.backend.reset_wait, - atomic_read(&kbdev->hwaccess.backend.reset_gpu) == KBASE_RESET_GPU_NOT_PENDING); - - return 0; -} -KBASE_EXPORT_TEST_API(kbase_reset_gpu_wait); - -int kbase_reset_gpu_init(struct kbase_device *kbdev) -{ - kbdev->hwaccess.backend.reset_workq = alloc_workqueue("Mali reset workqueue", 0, 1); - if (kbdev->hwaccess.backend.reset_workq == NULL) - return -ENOMEM; - - INIT_WORK(&kbdev->hwaccess.backend.reset_work, kbasep_reset_timeout_worker); - - hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - kbdev->hwaccess.backend.reset_timer.function = kbasep_reset_timer_callback; - - return 0; -} - -void kbase_reset_gpu_term(struct kbase_device *kbdev) -{ - destroy_workqueue(kbdev->hwaccess.backend.reset_workq); -} - -static u64 kbasep_apply_limited_core_mask(const struct kbase_device *kbdev, const u64 affinity, - const u64 limited_core_mask) -{ - const u64 result = affinity & limited_core_mask; - -#ifdef CONFIG_MALI_VALHALL_DEBUG - dev_dbg(kbdev->dev, - "Limiting affinity due to BASE_JD_REQ_LIMITED_CORE_MASK from 0x%lx to 0x%lx (mask is 0x%lx)\n", - (unsigned long)affinity, (unsigned long)result, (unsigned long)limited_core_mask); -#else - CSTD_UNUSED(kbdev); -#endif - - return result; -} diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_internal.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_internal.h deleted file mode 100644 index 2dd651a096fd..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_internal.h +++ /dev/null @@ -1,140 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Job Manager backend-specific low-level APIs. - */ - -#ifndef _KBASE_JM_HWACCESS_H_ -#define _KBASE_JM_HWACCESS_H_ - -#include -#include -#include - -#include -#include - -/** - * kbase_job_done_slot() - Complete the head job on a particular job-slot - * @kbdev: Device pointer - * @s: Job slot - * @completion_code: Completion code of job reported by GPU - * @job_tail: Job tail address reported by GPU - * @end_timestamp: Timestamp of job completion - */ -void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, u64 job_tail, - ktime_t *end_timestamp); - -/** - * kbase_job_hw_submit() - Submit a job to the GPU - * @kbdev: Device pointer - * @katom: Atom to submit - * @js: Job slot to submit on - * - * The caller must check kbasep_jm_is_submit_slots_free() != false before - * calling this. - * - * The following locking conditions are made on the caller: - * - it must hold the hwaccess_lock - * - * Return: 0 if the job was successfully submitted to hardware, an error otherwise. - */ -int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, unsigned int js); - -#if !MALI_USE_CSF -/** - * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop - * on the specified atom - * @kbdev: Device pointer - * @js: Job slot to stop on - * @action: The action to perform, either JS_COMMAND_HARD_STOP or - * JS_COMMAND_SOFT_STOP - * @core_reqs: Core requirements of atom to stop - * @target_katom: Atom to stop - * - * The following locking conditions are made on the caller: - * - it must hold the hwaccess_lock - */ -void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, unsigned int js, - u32 action, base_jd_core_req core_reqs, - struct kbase_jd_atom *target_katom); -#endif /* !MALI_USE_CSF */ - -/** - * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job - * slot belonging to a given context. - * @kbdev: Device pointer - * @kctx: Context pointer. May be NULL - * @katom: Specific atom to stop. May be NULL - * @js: Job slot to hard stop - * @action: The action to perform, either JS_COMMAND_HARD_STOP or - * JS_COMMAND_SOFT_STOP - * - * If no context is provided then all jobs on the slot will be soft or hard - * stopped. - * - * If a katom is provided then only that specific atom will be stopped. In this - * case the kctx parameter is ignored. - * - * Jobs that are on the slot but are not yet on the GPU will be unpulled and - * returned to the job scheduler. - * - * Return: true if an atom was stopped, false otherwise - */ -bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js, struct kbase_jd_atom *katom, u32 action); - -/** - * kbase_job_slot_init - Initialise job slot framework - * @kbdev: Device pointer - * - * Called on driver initialisation - * - * Return: 0 on success - */ -int kbase_job_slot_init(struct kbase_device *kbdev); - -/** - * kbase_job_slot_halt - Halt the job slot framework - * @kbdev: Device pointer - * - * Should prevent any further job slot processing - */ -void kbase_job_slot_halt(struct kbase_device *kbdev); - -/** - * kbase_job_slot_term - Terminate job slot framework - * @kbdev: Device pointer - * - * Called on driver termination - */ -void kbase_job_slot_term(struct kbase_device *kbdev); - -/** - * kbase_gpu_cache_clean - Cause a GPU cache clean & flush - * @kbdev: Device pointer - * - * Caller must not be in IRQ context - */ -void kbase_gpu_cache_clean(struct kbase_device *kbdev); - -#endif /* _KBASE_JM_HWACCESS_H_ */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.c deleted file mode 100644 index b211dd9748ca..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.c +++ /dev/null @@ -1,1793 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Register-based HW access backend specific APIs - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) -#include -#endif -#include -#include -#include -#include -#include - -/** - * SLOT_RB_EMPTY - Return whether the specified ringbuffer is empty. - * - * @rb: ring buffer - * - * Note: HW access lock must be held - */ -#define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) - -/** - * SLOT_RB_ENTRIES - Return number of atoms currently in the specified ringbuffer. - * - * @rb: ring buffer - * - * Note: HW access lock must be held - */ -#define SLOT_RB_ENTRIES(rb) ((int)(s8)(rb->write_idx - rb->read_idx)) - -static void kbase_gpu_release_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - ktime_t *end_timestamp); - -/** - * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer - * @kbdev: Device pointer - * @katom: Atom to enqueue - * - * Context: Caller must hold the HW access lock - */ -static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; - - WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; - rb->write_idx++; - - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; -} - -/** - * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once - * it has been completed - * @kbdev: Device pointer - * @js: Job slot to remove atom from - * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in - * which case current time will be used. - * - * Context: Caller must hold the HW access lock - * - * Return: Atom removed from ringbuffer - */ -static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, unsigned int js, - ktime_t *end_timestamp) -{ - struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; - struct kbase_jd_atom *katom; - - if (SLOT_RB_EMPTY(rb)) { - WARN(1, "GPU ringbuffer unexpectedly empty\n"); - return NULL; - } - - lockdep_assert_held(&kbdev->hwaccess_lock); - - katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; - - kbase_gpu_release_atom(kbdev, katom, end_timestamp); - - rb->read_idx++; - - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; - - return katom; -} - -struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, unsigned int js, int idx) -{ - struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if ((SLOT_RB_ENTRIES(rb) - 1) < idx) - return NULL; /* idx out of range */ - - return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; -} - -struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, unsigned int js) -{ - struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; - - if (SLOT_RB_EMPTY(rb)) - return NULL; - - return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; -} - -bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) -{ - unsigned int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - int i; - - for (i = 0; i < SLOT_RB_SIZE; i++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); - - if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) - return true; - } - } - return false; -} - -u32 kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, unsigned int js) -{ - u32 nr = 0; - int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < SLOT_RB_SIZE; i++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); - - if (katom && (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)) - nr++; - } - - return nr; -} - -int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, unsigned int js) -{ - int nr = 0; - int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < SLOT_RB_SIZE; i++) { - if (kbase_gpu_inspect(kbdev, js, i)) - nr++; - } - - return nr; -} - -static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, unsigned int js, - enum kbase_atom_gpu_rb_state min_rb_state) -{ - int nr = 0; - int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < SLOT_RB_SIZE; i++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); - - if (katom && (katom->gpu_rb_state >= min_rb_state)) - nr++; - } - - return nr; -} - -/** - * check_secure_atom - Check if the given atom is in the given secure state and - * has a ringbuffer state of at least - * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION - * @katom: Atom pointer - * @secure: Desired secure state - * - * Return: true if atom is in the given state, false otherwise - */ -static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) -{ - if (katom->gpu_rb_state >= KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && - ((kbase_jd_katom_is_protected(katom) && secure) || - (!kbase_jd_katom_is_protected(katom) && !secure))) - return true; - - return false; -} - -/** - * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given - * secure state in the ringbuffers of at least - * state - * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE - * @kbdev: Device pointer - * @secure: Desired secure state - * - * Return: true if any atoms are in the given state, false otherwise - */ -static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, bool secure) -{ - unsigned int js; - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - int i; - - for (i = 0; i < SLOT_RB_SIZE; i++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); - - if (katom) { - if (check_secure_atom(katom, secure)) - return true; - } - } - } - - return false; -} - -int kbase_backend_slot_free(struct kbase_device *kbdev, unsigned int js) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_NOT_PENDING) { - /* The GPU is being reset - so prevent submission */ - return 0; - } - - return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); -} - -/** - * trace_atom_completion_for_gpu_metrics - Report the completion of atom for the - * purpose of emitting power/gpu_work_period - * tracepoint. - * - * @katom: Pointer to the atom that completed execution on GPU. - * @end_timestamp: Pointer to the timestamp of atom completion. May be NULL, in - * which case current time will be used. - * - * The function would also report the start for an atom that was in the HEAD_NEXT - * register. - * - * Note: Caller must hold the HW access lock. - */ -static inline void trace_atom_completion_for_gpu_metrics(struct kbase_jd_atom *const katom, - ktime_t *end_timestamp) -{ -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - u64 complete_ns; - struct kbase_context *kctx = katom->kctx; - struct kbase_jd_atom *queued = kbase_gpu_inspect(kctx->kbdev, katom->slot_nr, 1); - -#ifdef CONFIG_MALI_VALHALL_DEBUG - WARN_ON(!kbase_gpu_inspect(kctx->kbdev, katom->slot_nr, 0)); -#endif - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - if (unlikely(queued == katom)) - return; - - /* A protected atom and a non-protected atom cannot be in the RB_SUBMITTED - * state at the same time in the job slot ringbuffer. Atom submission state - * machine prevents the submission of a non-protected atom until all - * protected atoms have completed and GPU has exited the protected mode. - * This implies that if the queued atom is in RB_SUBMITTED state, it shall - * be a protected atom and so we can return early. - */ - if (unlikely(kbase_jd_katom_is_protected(katom))) - return; - - if (likely(end_timestamp)) - complete_ns = ktime_to_ns(*end_timestamp); - else - complete_ns = ktime_get_raw_ns(); - - kbase_gpu_metrics_ctx_end_activity(kctx, complete_ns); - if (queued && queued->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) - kbase_gpu_metrics_ctx_start_activity(queued->kctx, complete_ns); -#else - CSTD_UNUSED(katom); - CSTD_UNUSED(end_timestamp); -#endif -} - -static void kbase_gpu_release_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - ktime_t *end_timestamp) -{ - struct kbase_context *kctx = katom->kctx; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - switch (katom->gpu_rb_state) { - case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: - /* Should be impossible */ - WARN(1, "Attempting to release atom not in ringbuffer\n"); - break; - - case KBASE_ATOM_GPU_RB_SUBMITTED: - trace_atom_completion_for_gpu_metrics(katom, end_timestamp); - kbase_kinstr_jm_atom_hw_release(katom); - /* Inform power management at start/finish of atom so it can - * update its GPU utilisation metrics. Mark atom as not - * submitted beforehand. - */ - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; - kbase_pm_metrics_update(kbdev, end_timestamp); - - /* Inform platform at start/finish of atom */ - kbasep_platform_event_atom_complete(katom); - - if (katom->core_req & BASE_JD_REQ_PERMON) - kbase_pm_release_gpu_cycle_counter_nolock(kbdev); - - KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, katom, - &kbdev->gpu_props.js_features[katom->slot_nr]); - KBASE_TLSTREAM_TL_NRET_ATOM_AS(kbdev, katom, &kbdev->as[kctx->as_nr]); - KBASE_TLSTREAM_TL_NRET_CTX_LPU(kbdev, kctx, - &kbdev->gpu_props.js_features[katom->slot_nr]); - - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_READY: - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: - break; - - case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: - if (kbase_jd_katom_is_protected(katom) && - (katom->protected_state.enter != KBASE_ATOM_ENTER_PROTECTED_CHECK) && - (katom->protected_state.enter != KBASE_ATOM_ENTER_PROTECTED_HWCNT)) { - kbase_pm_protected_override_disable(kbdev); - kbase_pm_update_cores_state_nolock(kbdev); - } - if (kbase_jd_katom_is_protected(katom) && - (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) - kbase_pm_protected_entry_override_disable(kbdev); - if (!kbase_jd_katom_is_protected(katom) && - (katom->protected_state.exit != KBASE_ATOM_EXIT_PROTECTED_CHECK) && - (katom->protected_state.exit != KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT)) { - kbase_pm_protected_override_disable(kbdev); - kbase_pm_update_cores_state_nolock(kbdev); - } - - if (katom->protected_state.enter != KBASE_ATOM_ENTER_PROTECTED_CHECK || - katom->protected_state.exit != KBASE_ATOM_EXIT_PROTECTED_CHECK) - kbdev->protected_mode_transition = false; - - /* If the atom is at KBASE_ATOM_ENTER_PROTECTED_HWCNT state, it means - * one of two events prevented it from progressing to the next state and - * ultimately reach protected mode: - * - hwcnts were enabled, and the atom had to schedule a worker to - * disable them. - * - the hwcnts were already disabled, but some other error occurred. - * In the first case, if the worker has not yet completed - * (kbdev->protected_mode_hwcnt_disabled == false), we need to re-enable - * them and signal to the worker they have already been enabled - */ - if (kbase_jd_katom_is_protected(katom) && - (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_HWCNT)) { - kbdev->protected_mode_hwcnt_desired = true; - if (kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbdev->protected_mode_hwcnt_disabled = false; - } - } - - /* If the atom has suspended hwcnt but has not yet entered - * protected mode, then resume hwcnt now. If the GPU is now in - * protected mode then hwcnt will be resumed by GPU reset so - * don't resume it here. - */ - if (kbase_jd_katom_is_protected(katom) && - ((katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_IDLE_L2) || - (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY) || - (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_FINISHED))) { - WARN_ON(!kbdev->protected_mode_hwcnt_disabled); - kbdev->protected_mode_hwcnt_desired = true; - if (kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbdev->protected_mode_hwcnt_disabled = false; - } - } - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { - if (katom->atom_flags & KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) { - kbase_pm_protected_l2_override(kbdev, false); - katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; - } - } - - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_RETURN_TO_JS: - break; - } - - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; - katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; -} - -static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_MARK_FOR_RETURN_TO_JS, katom->kctx, katom, - katom->jc, katom->slot_nr, katom->event_code); - kbase_gpu_release_atom(kbdev, katom, NULL); - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; -} - -/** - * other_slots_busy - Determine if any job slots other than @js are currently - * running atoms - * @kbdev: Device pointer - * @js: Job slot - * - * Return: true if any slots other than @js are busy, false otherwise - */ -static inline bool other_slots_busy(struct kbase_device *kbdev, unsigned int js) -{ - unsigned int slot; - - for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) { - if (slot == js) - continue; - - if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot, KBASE_ATOM_GPU_RB_SUBMITTED)) - return true; - } - - return false; -} - -static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) -{ - return kbdev->protected_mode; -} - -static void kbase_gpu_disable_coherent(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* - * When entering into protected mode, we must ensure that the - * GPU is not operating in coherent mode as well. This is to - * ensure that no protected memory can be leaked. - */ - if (kbdev->system_coherency == COHERENCY_ACE) - kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); -} - -static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) -{ - int err = -EINVAL; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - WARN_ONCE(!kbdev->protected_ops, - "Cannot enter protected mode: protected callbacks not specified.\n"); - - if (kbdev->protected_ops) { - /* Switch GPU to protected mode */ - err = kbdev->protected_ops->protected_mode_enable(kbdev->protected_dev); - - if (err) { - dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", err); - } else { - kbdev->protected_mode = true; - kbase_ipa_protection_mode_switch_event(kbdev); - } - } - - return err; -} - -static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - WARN_ONCE(!kbdev->protected_ops, - "Cannot exit protected mode: protected callbacks not specified.\n"); - - if (!kbdev->protected_ops) - return -EINVAL; - - /* The protected mode disable callback will be called as part of reset - */ - return kbase_reset_gpu_silent(kbdev); -} - -static int kbase_jm_protected_entry(struct kbase_device *kbdev, struct kbase_jd_atom **katom, - int idx, unsigned int js) -{ - int err = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - err = kbase_gpu_protected_mode_enter(kbdev); - - /* - * Regardless of result before this call, we are no longer - * transitioning the GPU. - */ - - kbdev->protected_mode_transition = false; - kbase_pm_protected_override_disable(kbdev); - kbase_pm_update_cores_state_nolock(kbdev); - - KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev, kbdev); - if (err) { - /* - * Failed to switch into protected mode. - * - * At this point we expect: - * katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && - * katom->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_FINISHED - * ==> - * kbdev->protected_mode_hwcnt_disabled = false - */ - katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; - kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); - /* - * Only return if head atom or previous atom - * already removed - as atoms must be returned - * in order. - */ - if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); - kbase_jm_return_atom_to_js(kbdev, katom[idx]); - } - - return -EINVAL; - } - - /* - * Protected mode sanity checks. - */ - WARN(kbase_jd_katom_is_protected(katom[idx]) != kbase_gpu_in_protected_mode(kbdev), - "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", - kbase_jd_katom_is_protected(katom[idx]), kbase_gpu_in_protected_mode(kbdev)); - katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; - - return err; -} - -static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbase_jd_atom **katom, - int idx, unsigned int js) -{ - int err = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - switch (katom[idx]->protected_state.enter) { - case KBASE_ATOM_ENTER_PROTECTED_CHECK: - KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev, kbdev); - /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV - * should ensure that we are not already transitiong, and that - * there are no atoms currently on the GPU. - */ - WARN_ON(kbdev->protected_mode_transition); - WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); - /* If hwcnt is disabled, it means we didn't clean up correctly - * during last exit from protected mode. - */ - WARN_ON(kbdev->protected_mode_hwcnt_disabled); - - katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_HWCNT; - - kbdev->protected_mode_transition = true; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_ENTER_PROTECTED_HWCNT: - /* See if we can get away with disabling hwcnt atomically */ - kbdev->protected_mode_hwcnt_desired = false; - if (!kbdev->protected_mode_hwcnt_disabled) { - if (kbase_hwcnt_context_disable_atomic(kbdev->hwcnt_gpu_ctx)) - kbdev->protected_mode_hwcnt_disabled = true; - } - - /* We couldn't disable atomically, so kick off a worker */ - if (!kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_queue_work(kbdev->hwcnt_gpu_ctx, - &kbdev->protected_mode_hwcnt_disable_work); - return -EAGAIN; - } - - /* Once reaching this point GPU must be switched to protected - * mode or hwcnt re-enabled. - */ - - if (kbase_pm_protected_entry_override_enable(kbdev)) - return -EAGAIN; - - /* - * Not in correct mode, begin protected mode switch. - * Entering protected mode requires us to power down the L2, - * and drop out of fully coherent mode. - */ - katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; - - kbase_pm_protected_override_enable(kbdev); - /* - * Only if the GPU reset hasn't been initiated, there is a need - * to invoke the state machine to explicitly power down the - * shader cores and L2. - */ - if (!kbdev->pm.backend.protected_entry_transition_override) - kbase_pm_update_cores_state_nolock(kbdev); - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: - /* Avoid unnecessary waiting on non-ACE platforms. */ - if (kbdev->system_coherency == COHERENCY_ACE) { - if (kbdev->pm.backend.l2_always_on) { - /* - * If the GPU reset hasn't completed, then L2 - * could still be powered up. - */ - if (kbase_reset_gpu_is_active(kbdev)) - return -EAGAIN; - } - - if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || - kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2) || - !kbase_io_has_gpu(kbdev)) { - /* - * The L2 is still powered, wait for all - * the users to finish with it before doing - * the actual reset. - */ - return -EAGAIN; - } - } - - katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: - /* - * When entering into protected mode, we must ensure that the - * GPU is not operating in coherent mode as well. This is to - * ensure that no protected memory can be leaked. - */ - kbase_gpu_disable_coherent(kbdev); - - kbase_pm_protected_entry_override_disable(kbdev); - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { - /* - * Power on L2 caches; this will also result in the - * correct value written to coherency enable register. - */ - kbase_pm_protected_l2_override(kbdev, true); - - /* - * Set the flag on the atom that additional - * L2 references are taken. - */ - katom[idx]->atom_flags |= KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; - } - - katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_FINISHED; - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) - return -EAGAIN; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_ENTER_PROTECTED_FINISHED: - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { - /* - * Check that L2 caches are powered and, if so, - * enter protected mode. - */ - if (kbdev->pm.backend.l2_state == KBASE_L2_ON) { - /* - * Remove additional L2 reference and reset - * the atom flag which denotes it. - */ - if (katom[idx]->atom_flags & KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) { - kbase_pm_protected_l2_override(kbdev, false); - katom[idx]->atom_flags &= - ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; - } - - err = kbase_jm_protected_entry(kbdev, katom, idx, js); - - if (err) - return err; - } else { - /* - * still waiting for L2 caches to power up - */ - return -EAGAIN; - } - } else { - err = kbase_jm_protected_entry(kbdev, katom, idx, js); - - if (err) - return err; - } - } - - return 0; -} - -static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, struct kbase_jd_atom **katom, - int idx, unsigned int js) -{ - int err = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - switch (katom[idx]->protected_state.exit) { - case KBASE_ATOM_EXIT_PROTECTED_CHECK: - KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev, kbdev); - /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV - * should ensure that we are not already transitiong, and that - * there are no atoms currently on the GPU. - */ - WARN_ON(kbdev->protected_mode_transition); - WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); - - /* - * Exiting protected mode requires a reset, but first the L2 - * needs to be powered down to ensure it's not active when the - * reset is issued. - */ - katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; - - kbdev->protected_mode_transition = true; - kbase_pm_protected_override_enable(kbdev); - kbase_pm_update_cores_state_nolock(kbdev); - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: - if (kbdev->pm.backend.l2_state != KBASE_L2_OFF) { - /* - * The L2 is still powered, wait for all the users to - * finish with it before doing the actual reset. - */ - return -EAGAIN; - } - katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_RESET; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_EXIT_PROTECTED_RESET: - /* L2 cache has been turned off (which is needed prior to the reset of GPU - * to exit the protected mode), so the override flag can be safely cleared. - * Even if L2 cache is powered up again before the actual reset, it should - * not be an issue (there are no jobs running on the GPU). - */ - kbase_pm_protected_override_disable(kbdev); - - /* Issue the reset to the GPU */ - err = kbase_gpu_protected_mode_reset(kbdev); - - if (err == -EAGAIN) - return -EAGAIN; - - if (err) { - kbdev->protected_mode_transition = false; - - /* Failed to exit protected mode, fail atom */ - katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; - kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); - /* Only return if head atom or previous atom - * already removed - as atoms must be returned in order - */ - if (idx == 0 || - katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); - kbase_jm_return_atom_to_js(kbdev, katom[idx]); - } - - /* If we're exiting from protected mode, hwcnt must have - * been disabled during entry. - */ - WARN_ON(!kbdev->protected_mode_hwcnt_disabled); - kbdev->protected_mode_hwcnt_desired = true; - if (kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbdev->protected_mode_hwcnt_disabled = false; - } - - return -EINVAL; - } - - katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: - /* A GPU reset is issued when exiting protected mode. Once the - * reset is done all atoms' state will also be reset. For this - * reason, if the atom is still in this state we can safely - * say that the reset has not completed i.e., we have not - * finished exiting protected mode yet. - */ - return -EAGAIN; - } - - return 0; -} - -void kbase_backend_slot_update(struct kbase_device *kbdev) -{ - unsigned int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (kbase_reset_gpu_is_active(kbdev) || !kbase_io_has_gpu(kbdev)) - return; - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - struct kbase_jd_atom *katom[2]; - int idx; - - katom[0] = kbase_gpu_inspect(kbdev, js, 0); - katom[1] = kbase_gpu_inspect(kbdev, js, 1); - WARN_ON(katom[1] && !katom[0]); - - for (idx = 0; idx < SLOT_RB_SIZE; idx++) { - bool cores_ready; -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - bool trace_atom_submit_for_gpu_metrics = true; -#endif - int ret; - - if (!katom[idx]) - continue; - - switch (katom[idx]->gpu_rb_state) { - case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: - /* Should be impossible */ - WARN(1, "Attempting to update atom not in ringbuffer\n"); - break; - - case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: - if (katom[idx]->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) - break; - - katom[idx]->gpu_rb_state = - KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: - if (kbase_gpu_check_secure_atoms( - kbdev, !kbase_jd_katom_is_protected(katom[idx]))) - break; - - if ((idx == 1) && (kbase_jd_katom_is_protected(katom[0]) != - kbase_jd_katom_is_protected(katom[1]))) - break; - - if (kbdev->protected_mode_transition) - break; - - katom[idx]->gpu_rb_state = - KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: - - /* - * Exiting protected mode must be done before - * the references on the cores are taken as - * a power down the L2 is required which - * can't happen after the references for this - * atom are taken. - */ - - if (!kbase_gpu_in_protected_mode(kbdev) && - kbase_jd_katom_is_protected(katom[idx])) { - /* Atom needs to transition into protected mode. */ - ret = kbase_jm_enter_protected_mode(kbdev, katom, idx, js); - if (ret) - break; - } else if (kbase_gpu_in_protected_mode(kbdev) && - !kbase_jd_katom_is_protected(katom[idx])) { - /* Atom needs to transition out of protected mode. */ - ret = kbase_jm_exit_protected_mode(kbdev, katom, idx, js); - if (ret) - break; - } - katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; - - /* Atom needs no protected mode transition. */ - - katom[idx]->gpu_rb_state = - KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: - if (katom[idx]->will_fail_event_code) { - kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); - /* Set EVENT_DONE so this atom will be - * completed, not unpulled. - */ - katom[idx]->event_code = BASE_JD_EVENT_DONE; - /* Only return if head atom or previous - * atom already removed - as atoms must - * be returned in order. - */ - if (idx == 0 || katom[0]->gpu_rb_state == - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); - kbase_jm_return_atom_to_js(kbdev, katom[idx]); - } - break; - } - - cores_ready = kbase_pm_cores_requested(kbdev, true); - - if (!cores_ready) - break; - - katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_READY: - - if (idx == 1) { - enum kbase_atom_gpu_rb_state atom_0_gpu_rb_state = - katom[0]->gpu_rb_state; - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - trace_atom_submit_for_gpu_metrics = - (atom_0_gpu_rb_state == - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB); -#endif - - /* Only submit if head atom or previous - * atom already submitted - */ - if ((atom_0_gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED && - atom_0_gpu_rb_state != - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) - break; - - /* If intra-slot serialization in use - * then don't submit atom to NEXT slot - */ - if (kbdev->serialize_jobs & KBASE_SERIALIZE_INTRA_SLOT) - break; - } - - /* If inter-slot serialization in use then don't - * submit atom if any other slots are in use - */ - if ((kbdev->serialize_jobs & KBASE_SERIALIZE_INTER_SLOT) && - other_slots_busy(kbdev, js)) - break; - - /* Check if this job needs the cycle counter - * enabled before submission - */ - if (katom[idx]->core_req & BASE_JD_REQ_PERMON) - kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev); - - if (!kbase_job_hw_submit(kbdev, katom[idx], js)) { - katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_SUBMITTED; - - /* Inform power management at start/finish of - * atom so it can update its GPU utilisation - * metrics. - */ - kbase_pm_metrics_update(kbdev, - &katom[idx]->start_timestamp); - - /* Inform platform at start/finish of atom */ - kbasep_platform_event_atom_submit(katom[idx]); -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - if (likely(trace_atom_submit_for_gpu_metrics && - !kbase_jd_katom_is_protected(katom[idx]))) - kbase_gpu_metrics_ctx_start_activity( - katom[idx]->kctx, - ktime_to_ns(katom[idx]->start_timestamp)); -#endif - } else { - if (katom[idx]->core_req & BASE_JD_REQ_PERMON) - kbase_pm_release_gpu_cycle_counter_nolock(kbdev); - - break; - } - - /* ***TRANSITION TO HIGHER STATE*** */ - fallthrough; - case KBASE_ATOM_GPU_RB_SUBMITTED: - break; - - case KBASE_ATOM_GPU_RB_RETURN_TO_JS: - /* Only return if head atom or previous atom - * already removed - as atoms must be returned - * in order - */ - if (idx == 0 || - katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); - kbase_jm_return_atom_to_js(kbdev, katom[idx]); - } - break; - } - } - } -} - -void kbase_backend_run_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "Backend running atom %pK\n", (void *)katom); - - kbase_gpu_enqueue_atom(kbdev, katom); - kbase_backend_slot_update(kbdev); -} - -/** - * kbase_rb_atom_might_depend - determine if one atom in the slot ringbuffer - * might depend on another from the same kctx - * @katom_a: dependee atom - * @katom_b: atom to query - * - * This can be used on atoms that belong to different slot ringbuffers - * - * Return: true if @katom_b might depend on @katom_a, false if it cannot depend. - */ -static inline bool kbase_rb_atom_might_depend(const struct kbase_jd_atom *katom_a, - const struct kbase_jd_atom *katom_b) -{ - if (katom_a->kctx != katom_b->kctx) - return false; - return (katom_b->pre_dep || (katom_b->atom_flags & (KBASE_KATOM_FLAG_X_DEP_BLOCKED | - KBASE_KATOM_FLAG_FAIL_BLOCKER))); -} - -static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - u32 action, bool disjoint) -{ - struct kbase_context *kctx = katom->kctx; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; - kbase_gpu_mark_atom_for_return(kbdev, katom); - kbase_jsctx_slot_prio_blocked_set(kctx, katom->slot_nr, katom->sched_priority); - - if (disjoint) - kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); -} - -/** - * kbase_gpu_irq_evict - evict a slot's JS_HEAD_NEXT atom from the HW if it is - * related to a failed JS_HEAD atom - * @kbdev: kbase device - * @js: job slot to check - * @completion_code: completion code of the failed atom - * - * Note: 'STOPPED' atoms are considered 'failed', as they are in the HW, but - * unlike other failure codes we _can_ re-run them. - * - * This forms step 1 in a 2-step process of removing any related atoms from a - * slot's JS_HEAD_NEXT (ringbuffer index 1), should there have - * been a 'failure' on an atom in JS_HEAD (ringbuffer index 0). - * - * This step only removes the atoms from the HW, and marks them as - * (potentially) ready to run again. - * - * Step 2 is on marking the JS_HEAD atom as complete - * (kbase_gpu_complete_hw()), to dequeue said atoms and return them to the JS - * as appropriate, or re-submit them. - * - * Hence, this function must evict at a minimum the atoms related to the atom - * in JS_HEAD that kbase_gpu_complete_hw() will also dequeue. It is acceptable - * if this function evicts more atoms than kbase_gpu_complete_hw() dequeues, as - * the next kbase_backend_slot_update() will resubmit any remaining. - * - * Return: true if an atom was evicted, false otherwise. - */ -bool kbase_gpu_irq_evict(struct kbase_device *kbdev, unsigned int js, u32 completion_code) -{ - struct kbase_jd_atom *katom; - struct kbase_jd_atom *next_katom; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - katom = kbase_gpu_inspect(kbdev, js, 0); - if (!katom) { - dev_err(kbdev->dev, "Can't get a katom from js(%u)\n", js); - return false; - } - next_katom = kbase_gpu_inspect(kbdev, js, 1); - - if (next_katom && next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && - (kbase_rb_atom_might_depend(katom, next_katom) || - kbase_js_atom_runs_before(kbdev, katom, next_katom, 0u)) && - kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(js, HEAD_NEXT)) != 0) { - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), JS_COMMAND_NOP); - - if (completion_code == BASE_JD_EVENT_STOPPED) { - kbase_gpu_remove_atom(kbdev, next_katom, JS_COMMAND_SOFT_STOP, false); - KBASE_TLSTREAM_TL_NRET_ATOM_LPU( - kbdev, next_katom, - &kbdev->gpu_props.js_features[next_katom->slot_nr]); - KBASE_TLSTREAM_TL_NRET_ATOM_AS(kbdev, next_katom, - &kbdev->as[next_katom->kctx->as_nr]); - KBASE_TLSTREAM_TL_NRET_CTX_LPU( - kbdev, next_katom->kctx, - &kbdev->gpu_props.js_features[next_katom->slot_nr]); - } else { - next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; - - if (next_katom->core_req & BASE_JD_REQ_PERMON) - kbase_pm_release_gpu_cycle_counter_nolock(kbdev); - } - - /* On evicting the next_katom, the last submission kctx on the - * given job slot then reverts back to the one that owns katom. - * The aim is to enable the next submission that can determine - * if the read only shader core L1 cache should be invalidated. - */ - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = - SLOT_RB_TAG_KCTX(katom->kctx); - - return true; - } - - return false; -} - -/** - * kbase_gpu_complete_hw - complete the atom in a slot's JS_HEAD - * @kbdev: kbase device - * @js: job slot to check - * @completion_code: completion code of the completed atom - * @job_tail: value read from JS_TAIL, for STOPPED atoms - * @end_timestamp: pointer to approximate ktime value when the katom completed - * - * Among other operations, this also executes step 2 of a 2-step process of - * removing any related atoms from a slot's JS_HEAD_NEXT (ringbuffer index 1), - * should there have been a 'failure' on an atom in JS_HEAD (ringbuffer index - * 0). The first step is done in kbase_gpu_irq_evict(). - * - * Note: 'STOPPED' atoms are considered 'failed', as they are in the HW, but - * unlike other failure codes we _can_ re-run them. - * - * When the JS_HEAD atom is considered to be 'failed', then this will dequeue - * and return to the JS some (usually all) of the atoms evicted from the HW - * during the kbase_gpu_irq_evict() for that JS_HEAD atom. If it dequeues an - * atom, that atom must not have been running or must already be evicted, as - * otherwise we would be in the incorrect state of having an atom both running - * on the HW and returned to the JS. - */ - -void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 completion_code, - u64 job_tail, ktime_t *end_timestamp) -{ - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); - struct kbase_context *kctx = NULL; - - if (unlikely(!katom)) { - dev_err(kbdev->dev, "Can't get a katom from js(%d)\n", js); - return; - } - - kctx = katom->kctx; - - dev_dbg(kbdev->dev, "Atom %pK completed on hw with code 0x%x and job_tail 0x%llx (s:%d)\n", - (void *)katom, completion_code, job_tail, js); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* - * When a hard-stop is followed close after a soft-stop, the completion - * code may be set to STOPPED, even though the job is terminated - */ - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TMIX_8438)) { - if (completion_code == BASE_JD_EVENT_STOPPED && - (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { - completion_code = BASE_JD_EVENT_TERMINATED; - } - } - - if ((katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && - completion_code != BASE_JD_EVENT_DONE && !(completion_code & BASE_JD_SW_EVENT)) { - /* When a job chain fails, on a T60x or when - * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not - * flushed. To prevent future evictions causing possible memory - * corruption we need to flush the cache manually before any - * affected memory gets reused. - */ - katom->need_cache_flush_cores_retained = true; - } - - katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); - - if (completion_code == BASE_JD_EVENT_STOPPED) { - struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, 0); - - /* - * Dequeue next atom from ringbuffers on same slot if required. - * This atom will already have been removed from the NEXT - * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that - * the atoms on this slot are returned in the correct order. - */ - if (next_katom && kbase_js_atom_runs_before(kbdev, katom, next_katom, 0u)) { - WARN_ON(next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED); - kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); - kbase_jm_return_atom_to_js(kbdev, next_katom); - } - } else if (completion_code != BASE_JD_EVENT_DONE) { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - unsigned int i; - - if (!kbase_ctx_flag(katom->kctx, KCTX_DYING) && - !kbase_ctx_flag(katom->kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) { - dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", - js, completion_code, kbase_gpu_exception_name(completion_code)); - - } - -#if KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR != 0 - KBASE_KTRACE_DUMP(kbdev); -#endif - kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); - - /* - * Remove all atoms on the same context from ringbuffers. This - * will not remove atoms that are already on the GPU, as these - * are guaranteed not to have fail dependencies on the failed - * atom. - */ - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { - struct kbase_jd_atom *katom_idx0 = kbase_gpu_inspect(kbdev, i, 0); - struct kbase_jd_atom *katom_idx1 = kbase_gpu_inspect(kbdev, i, 1); - - if (katom_idx0 && kbase_rb_atom_might_depend(katom, katom_idx0) && - katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Dequeue katom_idx0 from ringbuffer */ - kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); - - if (katom_idx1 && kbase_rb_atom_might_depend(katom, katom_idx1) && - katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Dequeue katom_idx1 from ringbuffer */ - kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); - - katom_idx1->event_code = BASE_JD_EVENT_STOPPED; - kbase_jm_return_atom_to_js(kbdev, katom_idx1); - } - katom_idx0->event_code = BASE_JD_EVENT_STOPPED; - kbase_jm_return_atom_to_js(kbdev, katom_idx0); - - } else if (katom_idx1 && kbase_rb_atom_might_depend(katom, katom_idx1) && - katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Can not dequeue this atom yet - will be - * dequeued when atom at idx0 completes - */ - katom_idx1->event_code = BASE_JD_EVENT_STOPPED; - kbase_gpu_mark_atom_for_return(kbdev, katom_idx1); - } - } - } - - KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, js, - completion_code); - - if (job_tail != 0 && job_tail != katom->jc) { - /* Some of the job has been executed */ - dev_dbg(kbdev->dev, "Update job chain address of atom %pK to resume from 0x%llx\n", - (void *)katom, job_tail); - - /* Some of the job has been executed, so we update the job chain address to where - * we should resume from - */ - katom->jc = job_tail; - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, katom, job_tail, js); - } - - /* Only update the event code for jobs that weren't cancelled */ - if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) - katom->event_code = (enum base_jd_event_code)completion_code; - - /* Complete the job, and start new ones - * - * Also defer remaining work onto the workqueue: - * - Re-queue Soft-stopped jobs - * - For any other jobs, queue the job back into the dependency system - * - Schedule out the parent context if necessary, and schedule a new - * one in. - */ - if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) - kbase_reset_gpu_silent(kbdev); - - if (completion_code == BASE_JD_EVENT_STOPPED) - katom = kbase_jm_return_atom_to_js(kbdev, katom); - else - katom = kbase_jm_complete(kbdev, katom, end_timestamp); - - if (katom) { - dev_dbg(kbdev->dev, "Cross-slot dependency %pK has become runnable.\n", - (void *)katom); - - /* Cross-slot dependency has now become runnable. Try to submit it. */ - - /* Check if there are lower priority jobs to soft stop */ - kbase_job_slot_ctx_priority_check_locked(kctx, katom); - - kbase_jm_try_kick(kbdev, 1UL << katom->slot_nr); - } - - /* For partial shader core off L2 cache flush */ - kbase_pm_update_state(kbdev); - - /* Job completion may have unblocked other atoms. Try to update all job - * slots - */ - kbase_backend_slot_update(kbdev); -} - -void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) -{ - unsigned int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* Reset should always take the GPU out of protected mode */ - WARN_ON(kbase_gpu_in_protected_mode(kbdev)); - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - int atom_idx = 0; - int idx; - - for (idx = 0; idx < SLOT_RB_SIZE; idx++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, atom_idx); - bool keep_in_jm_rb = false; - - if (!katom) - break; - if (katom->protected_state.exit == KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) { - /* protected mode sanity checks */ - WARN(kbase_jd_katom_is_protected(katom) != - kbase_gpu_in_protected_mode(kbdev), - "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", - kbase_jd_katom_is_protected(katom), - kbase_gpu_in_protected_mode(kbdev)); - WARN(!(kbase_jd_katom_is_protected(katom) && js == 0) && - kbase_jd_katom_is_protected(katom), - "Protected atom on JS%u not supported", js); - } - if ((katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) && - !kbase_ctx_flag(katom->kctx, KCTX_DYING)) - keep_in_jm_rb = true; - - kbase_gpu_release_atom(kbdev, katom, NULL); - - /* - * If the atom wasn't on HW when the reset was issued - * then leave it in the RB and next time we're kicked - * it will be processed again from the starting state. - */ - if (kbase_io_has_gpu(kbdev) && keep_in_jm_rb) { - katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; - /* As the atom was not removed, increment the - * index so that we read the correct atom in the - * next iteration. - */ - atom_idx++; - continue; - } - - /* - * The atom was on the HW when the reset was issued - * all we can do is fail the atom. - */ - kbase_gpu_dequeue_atom(kbdev, js, NULL); - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - kbase_jm_complete(kbdev, katom, end_timestamp); - } - - /* Clear the slot's last katom submission kctx on reset */ - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_NULL_TAG_VAL; - } - - /* Re-enable GPU hardware counters if we're resetting from protected - * mode. - */ - kbdev->protected_mode_hwcnt_desired = true; - if (kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbdev->protected_mode_hwcnt_disabled = false; - - KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev, kbdev); - } - - kbdev->protected_mode_transition = false; - kbase_pm_protected_override_disable(kbdev); -} - -/** - * should_stop_next_atom - given a soft/hard stop action, determine if the next - * atom on a slot should be stopped - * @kbdev: kbase devices - * @head_katom: atom currently in the JS_HEAD - * @next_katom: atom currently in the JS_HEAD_NEXT - * @action: COMMAND_<...> action for soft/hard-stop - * - * This is used in cases where @head_katom is the target of the soft/hard-stop. - * It only makes sense to call this when @head_katom and @next_katom are from - * the same slot. - * - * Return: true if @next_katom should also be stopped with the given action, - * false otherwise - */ -static bool should_stop_next_atom(struct kbase_device *kbdev, - const struct kbase_jd_atom *head_katom, - const struct kbase_jd_atom *next_katom, u32 action) -{ - bool ret = false; - u32 hw_action = action & JS_COMMAND_MASK; - - switch (hw_action) { - case JS_COMMAND_SOFT_STOP: - ret = kbase_js_atom_runs_before(kbdev, head_katom, next_katom, 0u); - break; - case JS_COMMAND_HARD_STOP: - /* Unlike soft-stop, a hard-stop targeting a particular atom - * should not cause atoms from unrelated contexts to be - * removed - */ - ret = (head_katom->kctx == next_katom->kctx); - break; - default: - /* Other stop actions are possible, but the driver should not - * be generating them at this point in the call chain - */ - WARN(1, "Unexpected stop action: 0x%.8x", hw_action); - break; - } - return ret; -} - -static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, unsigned int js, - struct kbase_jd_atom *katom, u32 action) -{ - struct kbase_context *kctx = katom->kctx; - u32 hw_action = action & JS_COMMAND_MASK; - - kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); - kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, katom->core_req, katom); - kbase_jsctx_slot_prio_blocked_set(kctx, js, katom->sched_priority); -} - -static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) -{ - if (katom->x_post_dep) { - struct kbase_jd_atom *dep_atom = katom->x_post_dep; - - if (dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && - dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_RETURN_TO_JS) - return (int)dep_atom->slot_nr; - } - return -1; -} - -bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js, struct kbase_jd_atom *katom, u32 action) -{ - struct kbase_jd_atom *katom_idx0; - struct kbase_context *kctx_idx0 = NULL; - struct kbase_jd_atom *katom_idx1; - struct kbase_context *kctx_idx1 = NULL; - - bool katom_idx0_valid, katom_idx1_valid; - - bool ret = false; - - int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; - int prio_idx0 = 0, prio_idx1 = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); - katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); - - if (katom_idx0) { - kctx_idx0 = katom_idx0->kctx; - prio_idx0 = katom_idx0->sched_priority; - } - if (katom_idx1) { - kctx_idx1 = katom_idx1->kctx; - prio_idx1 = katom_idx1->sched_priority; - } - - if (katom) { - katom_idx0_valid = (katom_idx0 == katom); - if (katom_idx1) - katom_idx1_valid = (katom_idx1 == katom); - else - katom_idx1_valid = false; - } else { - katom_idx0_valid = (katom_idx0 && (!kctx || kctx_idx0 == kctx)); - katom_idx1_valid = (katom_idx1 && (!kctx || kctx_idx1 == kctx)); - } - /* If there's an atom in JS_HEAD_NEXT that we haven't already decided - * to stop, but we're stopping the JS_HEAD atom, see if they are - * related/ordered in some way that would require the same stop action - */ - if (!katom_idx1_valid && katom_idx0_valid && katom_idx1) - katom_idx1_valid = should_stop_next_atom(kbdev, katom_idx0, katom_idx1, action); - - if (katom_idx0_valid) - stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); - if (katom_idx1_valid) - stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); - - if (katom_idx0_valid) { - if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Simple case - just dequeue and return */ - kbase_gpu_dequeue_atom(kbdev, js, NULL); - if (katom_idx1_valid) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); - katom_idx1->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; - kbase_jm_return_atom_to_js(kbdev, katom_idx1); - kbase_jsctx_slot_prio_blocked_set(kctx_idx1, js, prio_idx1); - } - - katom_idx0->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; - kbase_jm_return_atom_to_js(kbdev, katom_idx0); - kbase_jsctx_slot_prio_blocked_set(kctx_idx0, js, prio_idx0); - } else { - /* katom_idx0 is on GPU */ - if (katom_idx1_valid && - katom_idx1->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) { - /* katom_idx0 and katom_idx1 are on GPU */ - - if (kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT)) == - 0) { - /* idx0 has already completed - stop - * idx1 if needed - */ - if (katom_idx1_valid) { - kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); - ret = true; - } - } else { - /* idx1 is in NEXT registers - attempt - * to remove - */ - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), - JS_COMMAND_NOP); - - if (kbase_reg_read64(kbdev, - JOB_SLOT_OFFSET(js, HEAD_NEXT)) != 0) { - /* idx1 removed successfully, - * will be handled in IRQ - */ - kbase_gpu_remove_atom(kbdev, katom_idx1, action, - true); - /* Revert the last_context. */ - kbdev->hwaccess.backend.slot_rb[js] - .last_kctx_tagged = - SLOT_RB_TAG_KCTX(katom_idx0->kctx); - - stop_x_dep_idx1 = - should_stop_x_dep_slot(katom_idx1); - - /* stop idx0 if still on GPU */ - kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); - ret = true; - } else if (katom_idx1_valid) { - /* idx0 has already completed, - * stop idx1 if needed - */ - kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); - ret = true; - } - } - } else if (katom_idx1_valid) { - /* idx1 not on GPU but must be dequeued*/ - - /* idx1 will be handled in IRQ */ - kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); - /* stop idx0 */ - /* This will be repeated for anything removed - * from the next registers, since their normal - * flow was also interrupted, and this function - * might not enter disjoint state e.g. if we - * don't actually do a hard stop on the head - * atom - */ - kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); - ret = true; - } else { - /* no atom in idx1 */ - /* just stop idx0 */ - kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); - ret = true; - } - } - } else if (katom_idx1_valid) { - if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Mark for return */ - /* idx1 will be returned once idx0 completes */ - kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); - } else { - /* idx1 is on GPU */ - if (kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT)) == 0) { - /* idx0 has already completed - stop idx1 */ - kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); - ret = true; - } else { - /* idx1 is in NEXT registers - attempt to - * remove - */ - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), - JS_COMMAND_NOP); - - if (kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(js, HEAD_NEXT)) != 0) { - /* idx1 removed successfully, will be - * handled in IRQ once idx0 completes - */ - kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); - /* Revert the last_context, or mark as purged */ - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = - kctx_idx0 ? SLOT_RB_TAG_KCTX(katom_idx0->kctx) : - SLOT_RB_TAG_PURGED; - } else { - /* idx0 has already completed - stop - * idx1 - */ - kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); - ret = true; - } - } - } - } - - if (stop_x_dep_idx0 != -1) - kbase_backend_soft_hard_stop_slot(kbdev, kctx, (unsigned int)stop_x_dep_idx0, NULL, - action); - - if (stop_x_dep_idx1 != -1) - kbase_backend_soft_hard_stop_slot(kbdev, kctx, (unsigned int)stop_x_dep_idx1, NULL, - action); - - return ret; -} - -void kbase_backend_cache_clean(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - if (katom->need_cache_flush_cores_retained) { - kbase_gpu_start_cache_clean(kbdev, GPU_COMMAND_CACHE_CLN_INV_FULL); - kbase_gpu_wait_cache_clean(kbdev); - - katom->need_cache_flush_cores_retained = false; - } -} - -void kbase_backend_complete_wq(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - /* - * If cache flush required due to HW workaround then perform the flush - * now - */ - kbase_backend_cache_clean(kbdev, katom); -} - -void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, base_jd_core_req core_req) -{ - CSTD_UNUSED(core_req); - - if (!kbdev->pm.active_count) { - kbase_pm_lock(kbdev); - kbase_pm_update_active(kbdev); - kbase_pm_unlock(kbdev); - } -} - -void kbase_gpu_dump_slots(struct kbase_device *kbdev) -{ - unsigned long flags; - unsigned int js; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - dev_info(kbdev->dev, "%s:\n", __func__); - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - int idx; - - for (idx = 0; idx < SLOT_RB_SIZE; idx++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, idx); - - if (katom) - dev_info(kbdev->dev, " js%u idx%d : katom=%pK gpu_rb_state=%d\n", - js, idx, katom, katom->gpu_rb_state); - else - dev_info(kbdev->dev, " js%u idx%d : empty\n", js, idx); - } - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -void kbase_backend_slot_kctx_purge_locked(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - unsigned int js; - bool tracked = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - u64 tagged_kctx = kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged; - - if (tagged_kctx == SLOT_RB_TAG_KCTX(kctx)) { - /* Marking the slot kctx tracking field is purged */ - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_TAG_PURGED; - tracked = true; - } - } - - if (tracked) { - /* The context had run some jobs before the purge, other slots - * in SLOT_RB_NULL_TAG_VAL condition needs to be marked as - * purged as well. - */ - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged == - SLOT_RB_NULL_TAG_VAL) - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = - SLOT_RB_TAG_PURGED; - } - } -} diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.h deleted file mode 100644 index 32be0bf44655..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_jm_rb.h +++ /dev/null @@ -1,77 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2018, 2020-2022 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Register-based HW access backend specific APIs - */ - -#ifndef _KBASE_HWACCESS_GPU_H_ -#define _KBASE_HWACCESS_GPU_H_ - -#include - -/** - * kbase_gpu_irq_evict - Evict an atom from a NEXT slot - * - * @kbdev: Device pointer - * @js: Job slot to evict from - * @completion_code: Event code from job that was run. - * - * Evict the atom in the NEXT slot for the specified job slot. This function is - * called from the job complete IRQ handler when the previous job has failed. - * - * Return: true if job evicted from NEXT registers, false otherwise - */ -bool kbase_gpu_irq_evict(struct kbase_device *kbdev, unsigned int js, u32 completion_code); - -/** - * kbase_gpu_complete_hw - Complete an atom on job slot js - * - * @kbdev: Device pointer - * @js: Job slot that has completed - * @completion_code: Event code from job that has completed - * @job_tail: The tail address from the hardware if the job has partially - * completed - * @end_timestamp: Time of completion - */ -void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 completion_code, - u64 job_tail, ktime_t *end_timestamp); - -/** - * kbase_gpu_inspect - Inspect the contents of the HW access ringbuffer - * - * @kbdev: Device pointer - * @js: Job slot to inspect - * @idx: Index into ringbuffer. 0 is the job currently running on - * the slot, 1 is the job waiting, all other values are invalid. - * Return: The atom at that position in the ringbuffer - * or NULL if no atom present - */ -struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, unsigned int js, int idx); - -/** - * kbase_gpu_dump_slots - Print the contents of the slot ringbuffers - * - * @kbdev: Device pointer - */ -void kbase_gpu_dump_slots(struct kbase_device *kbdev); - -#endif /* _KBASE_HWACCESS_GPU_H_ */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_backend.c deleted file mode 100644 index dd0d915be48c..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_backend.c +++ /dev/null @@ -1,371 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Register-based HW access backend specific job scheduler APIs - */ - -#include -#include -#include -#include -#include -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) -#include -#endif - -/* - * Hold the runpool_mutex for this - */ -static inline bool timer_callback_should_run(struct kbase_device *kbdev, int nr_running_ctxs) -{ - lockdep_assert_held(&kbdev->js_data.runpool_mutex); - -#ifdef CONFIG_MALI_VALHALL_DEBUG - if (kbdev->js_data.softstop_always) { - /* Debug support for allowing soft-stop on a single context */ - return true; - } -#endif /* CONFIG_MALI_VALHALL_DEBUG */ - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_9435)) { - /* Timeouts would have to be 4x longer (due to micro- - * architectural design) to support OpenCL conformance tests, so - * only run the timer when there's: - * - 2 or more CL contexts - * - 1 or more GLES contexts - * - * NOTE: We will treat a context that has both Compute and Non- - * Compute jobs will be treated as an OpenCL context (hence, we - * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). - */ - { - int nr_compute_ctxs = kbasep_js_ctx_attr_count_on_runpool( - kbdev, KBASEP_JS_CTX_ATTR_COMPUTE); - int nr_noncompute_ctxs = nr_running_ctxs - nr_compute_ctxs; - - return (bool)(nr_compute_ctxs >= 2 || nr_noncompute_ctxs > 0); - } - } else { - /* Run the timer callback whenever you have at least 1 context - */ - return (bool)(nr_running_ctxs > 0); - } -} - -static enum hrtimer_restart timer_callback(struct hrtimer *timer) -{ - unsigned long flags; - struct kbase_device *kbdev; - struct kbasep_js_device_data *js_devdata; - struct kbase_backend_data *backend; - unsigned int s; - bool reset_needed = false; - - KBASE_DEBUG_ASSERT(timer != NULL); - - backend = container_of(timer, struct kbase_backend_data, scheduling_timer); - kbdev = container_of(backend, struct kbase_device, hwaccess.backend); - js_devdata = &kbdev->js_data; - - /* Loop through the slots */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) { - struct kbase_jd_atom *atom = NULL; - - if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) { - atom = kbase_gpu_inspect(kbdev, s, 0); - KBASE_DEBUG_ASSERT(atom != NULL); - } - - if (atom != NULL) { - /* The current version of the model doesn't support - * Soft-Stop - */ - if (!kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_5736)) { - u32 ticks = atom->ticks++; - -#if !defined(CONFIG_MALI_VALHALL_JOB_DUMP) && !defined(CONFIG_MALI_VECTOR_DUMP) - u32 soft_stop_ticks, hard_stop_ticks, gpu_reset_ticks; - if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { - soft_stop_ticks = js_devdata->soft_stop_ticks_cl; - hard_stop_ticks = js_devdata->hard_stop_ticks_cl; - gpu_reset_ticks = js_devdata->gpu_reset_ticks_cl; - } else { - soft_stop_ticks = js_devdata->soft_stop_ticks; - if (kbase_is_quick_reset_enabled(kbdev)) { - hard_stop_ticks = 2; - gpu_reset_ticks = 3; - } else { - hard_stop_ticks = js_devdata->hard_stop_ticks_ss; - gpu_reset_ticks = js_devdata->gpu_reset_ticks_ss; - } - } - - /* If timeouts have been changed then ensure - * that atom tick count is not greater than the - * new soft_stop timeout. This ensures that - * atoms do not miss any of the timeouts due to - * races between this worker and the thread - * changing the timeouts. - */ - if (backend->timeouts_updated && ticks > soft_stop_ticks) - ticks = atom->ticks = soft_stop_ticks; - - /* Job is Soft-Stoppable */ - if (ticks == soft_stop_ticks) { - /* Job has been scheduled for at least - * js_devdata->soft_stop_ticks ticks. - * Soft stop the slot so we can run - * other jobs. - */ -#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS - int disjoint_threshold = - KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD; - u32 softstop_flags = 0u; - - dev_dbg(kbdev->dev, "Soft-stop"); - /* nr_user_contexts_running is updated - * with the runpool_mutex, but we can't - * take that here. - * - * However, if it's about to be - * increased then the new context can't - * run any jobs until they take the - * hwaccess_lock, so it's OK to observe - * the older value. - * - * Similarly, if it's about to be - * decreased, the last job from another - * context has already finished, so - * it's not too bad that we observe the - * older value and register a disjoint - * event when we try soft-stopping - */ - if (js_devdata->nr_user_contexts_running >= - disjoint_threshold) - softstop_flags |= JS_COMMAND_SW_CAUSES_DISJOINT; - - kbase_job_slot_softstop_swflags(kbdev, s, atom, - softstop_flags); -#endif - } else if (ticks == hard_stop_ticks) { - /* Job has been scheduled for at least - * js_devdata->hard_stop_ticks_ss ticks. - * It should have been soft-stopped by - * now. Hard stop the slot. - */ -#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS - u32 ms = js_devdata->scheduling_period_ns / 1000000u; - if (!kbase_is_quick_reset_enabled(kbdev)) - dev_warn( - kbdev->dev, - "JS: Job Hard-Stopped (took more than %u ticks at %u ms/tick)", - ticks, ms); - kbase_job_slot_hardstop(atom->kctx, s, atom); -#endif - } else if (ticks == gpu_reset_ticks) { - /* Job has been scheduled for at least - * js_devdata->gpu_reset_ticks_ss ticks. - * It should have left the GPU by now. - * Signal that the GPU needs to be - * reset. - */ - reset_needed = true; - } -#else /* !CONFIG_MALI_VALHALL_JOB_DUMP */ - /* NOTE: During CONFIG_MALI_VALHALL_JOB_DUMP, we use - * the alternate timeouts, which makes the hard- - * stop and GPU reset timeout much longer. We - * also ensure that we don't soft-stop at all. - */ - if (ticks == js_devdata->soft_stop_ticks) { - /* Job has been scheduled for at least - * js_devdata->soft_stop_ticks. We do - * not soft-stop during - * CONFIG_MALI_VALHALL_JOB_DUMP, however. - */ - dev_dbg(kbdev->dev, "Soft-stop"); - } else if (ticks == js_devdata->hard_stop_ticks_dumping) { - /* Job has been scheduled for at least - * js_devdata->hard_stop_ticks_dumping - * ticks. Hard stop the slot. - */ -#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS - u32 ms = js_devdata->scheduling_period_ns / 1000000u; - dev_warn( - kbdev->dev, - "JS: Job Hard-Stopped (took more than %u ticks at %u ms/tick)", - ticks, ms); - kbase_job_slot_hardstop(atom->kctx, s, atom); -#endif - } else if (ticks == js_devdata->gpu_reset_ticks_dumping) { - /* Job has been scheduled for at least - * js_devdata->gpu_reset_ticks_dumping - * ticks. It should have left the GPU by - * now. Signal that the GPU needs to be - * reset. - */ - reset_needed = true; - } -#endif /* !CONFIG_MALI_VALHALL_JOB_DUMP */ - } - } - } - if (reset_needed) { - if (kbase_is_quick_reset_enabled(kbdev)) - dev_err(kbdev->dev, "quick reset"); - else - dev_err(kbdev->dev, - "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issuing GPU soft-reset to resolve."); - - if (kbase_prepare_to_reset_gpu_locked(kbdev, RESET_FLAGS_NONE)) - kbase_reset_gpu_locked(kbdev); - } - /* the timer is re-issued if there is contexts in the run-pool */ - - if (backend->timer_running) - hrtimer_start(&backend->scheduling_timer, - HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), - HRTIMER_MODE_REL); - - backend->timeouts_updated = false; - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return HRTIMER_NORESTART; -} - -void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) -{ - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - unsigned long flags; - /* Timer must stop if we are suspending */ - const bool suspend_timer = backend->suspend_timer; - const int nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); - - lockdep_assert_held(&js_devdata->runpool_mutex); - - if (suspend_timer || !timer_callback_should_run(kbdev, nr_running_ctxs)) { - /* Take spinlock to force synchronisation with timer */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - backend->timer_running = false; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - /* From now on, return value of timer_callback_should_run() - * will also cause the timer to not requeue itself. Its return - * value cannot change, because it depends on variables updated - * with the runpool_mutex held, which the caller of this must - * also hold - */ - hrtimer_cancel(&backend->scheduling_timer); - } - - if (!suspend_timer && timer_callback_should_run(kbdev, nr_running_ctxs) && - !backend->timer_running) { - /* Take spinlock to force synchronisation with timer */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - backend->timer_running = true; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - hrtimer_start(&backend->scheduling_timer, - HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), - HRTIMER_MODE_REL); - - KBASE_KTRACE_ADD_JM(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u); - } - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - if (unlikely(suspend_timer)) { - js_devdata->gpu_metrics_timer_needed = false; - /* Cancel the timer as System suspend is happening */ - hrtimer_cancel(&js_devdata->gpu_metrics_timer); - js_devdata->gpu_metrics_timer_running = false; - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - /* Explicitly emit the tracepoint on System suspend */ - kbase_gpu_metrics_emit_tracepoint(kbdev, ktime_get_raw_ns()); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return; - } - - if (!nr_running_ctxs) { - /* Just set the flag to not restart the timer on expiry */ - js_devdata->gpu_metrics_timer_needed = false; - return; - } - - /* There are runnable contexts so the timer is needed */ - if (!js_devdata->gpu_metrics_timer_needed) { - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - js_devdata->gpu_metrics_timer_needed = true; - /* No need to restart the timer if it is already running. */ - if (!js_devdata->gpu_metrics_timer_running) { - hrtimer_start(&js_devdata->gpu_metrics_timer, - HR_TIMER_DELAY_NSEC(kbase_gpu_metrics_get_tp_emit_interval()), - HRTIMER_MODE_REL); - js_devdata->gpu_metrics_timer_running = true; - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } -#endif -} - -int kbase_backend_timer_init(struct kbase_device *kbdev) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - backend->scheduling_timer.function = timer_callback; - backend->timer_running = false; - - return 0; -} - -void kbase_backend_timer_term(struct kbase_device *kbdev) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - hrtimer_cancel(&backend->scheduling_timer); -} - -void kbase_backend_timer_suspend(struct kbase_device *kbdev) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - backend->suspend_timer = true; - - kbase_backend_ctx_count_changed(kbdev); -} - -void kbase_backend_timer_resume(struct kbase_device *kbdev) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - backend->suspend_timer = false; - - kbase_backend_ctx_count_changed(kbdev); -} - -void kbase_backend_timeouts_changed(struct kbase_device *kbdev) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - backend->timeouts_updated = true; -} diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_internal.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_internal.h deleted file mode 100644 index 4f7c371a1f1a..000000000000 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_js_internal.h +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2015, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Register-based HW access backend specific job scheduler APIs - */ - -#ifndef _KBASE_JS_BACKEND_H_ -#define _KBASE_JS_BACKEND_H_ - -/** - * kbase_backend_timer_init() - Initialise the JS scheduling timer - * @kbdev: Device pointer - * - * This function should be called at driver initialisation - * - * Return: 0 on success - */ -int kbase_backend_timer_init(struct kbase_device *kbdev); - -/** - * kbase_backend_timer_term() - Terminate the JS scheduling timer - * @kbdev: Device pointer - * - * This function should be called at driver termination - */ -void kbase_backend_timer_term(struct kbase_device *kbdev); - -/** - * kbase_backend_timer_suspend - Suspend is happening, stop the JS scheduling - * timer - * @kbdev: Device pointer - * - * This function should be called on suspend, after the active count has reached - * zero. This is required as the timer may have been started on job submission - * to the job scheduler, but before jobs are submitted to the GPU. - * - * Caller must hold runpool_mutex. - */ -void kbase_backend_timer_suspend(struct kbase_device *kbdev); - -/** - * kbase_backend_timer_resume - Resume is happening, re-evaluate the JS - * scheduling timer - * @kbdev: Device pointer - * - * This function should be called on resume. Note that is not guaranteed to - * re-start the timer, only evalute whether it should be re-started. - * - * Caller must hold runpool_mutex. - */ -void kbase_backend_timer_resume(struct kbase_device *kbdev); - -#endif /* _KBASE_JS_BACKEND_H_ */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.c index 527d0c5717e6..d752f2d9a6da 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.c @@ -36,7 +36,6 @@ #include -#if MALI_USE_CSF #include /* Index of the last value register for each type of core, with the 1st value @@ -55,11 +54,8 @@ static u32 gpu_control_base_addr; /* Array for storing the value of SELECT register for each type of core */ static u64 ipa_ctl_select_config[KBASE_IPA_CORE_TYPE_NUM]; static u32 ipa_control_timer_enabled; -#endif -#if MALI_USE_CSF static u32 sysc_alloc_regs[SYSC_ALLOC_COUNT]; -#endif #define LO_MASK(M) ((M)&0xFFFFFFFF) #define HI_MASK(M) ((M)&0xFFFFFFFF00000000) @@ -69,13 +65,8 @@ static u32 sysc_alloc_regs[SYSC_ALLOC_COUNT]; * significant bits, which are set to THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_SOFTWARE in * midgard_model_read_reg(). */ -#if MALI_USE_CSF #define THREAD_FEATURES_PARTIAL(MAX_REGISTERS, MAX_TASK_QUEUE, MAX_TG_SPLIT) \ ((MAX_REGISTERS) | ((MAX_TASK_QUEUE) << 24)) -#else -#define THREAD_FEATURES_PARTIAL(MAX_REGISTERS, MAX_TASK_QUEUE, MAX_TG_SPLIT) \ - ((MAX_REGISTERS) | ((MAX_TASK_QUEUE) << 16) | ((MAX_TG_SPLIT) << 24)) -#endif struct error_status_t hw_error_status; @@ -133,24 +124,18 @@ enum pwr_on_index { INDEX_TILER, INDEX_SHADER, INDEX_STACK, -#if MALI_USE_CSF INDEX_BASE, INDEX_NEURAL, -#endif INDEX_DOMAIN_COUNT }; struct dummy_model_t { int reset_completed; int reset_completed_mask; -#if !MALI_USE_CSF - int prfcnt_sample_completed; -#endif /* !MALI_USE_CSF */ int power_changed_mask; /* 2 bits: _ALL,_SINGLE */ int power_changed; /* 1 bit */ bool clean_caches_completed; bool clean_caches_completed_irq_enabled; -#if MALI_USE_CSF bool flush_pa_range_completed; bool flush_pa_range_completed_irq_enabled; /* Representations of COMMAND_NOT_ALLOWED and COMMAND_INVALID bits in @@ -164,7 +149,6 @@ struct dummy_model_t { u64 command_arg; /* PWR_CMDARG register */ u64 gov_core_mask; -#endif uint32_t domain_power_on[INDEX_DOMAIN_COUNT]; u32 coherency_enable; unsigned int job_irq_js_state; @@ -462,9 +446,6 @@ static const struct control_reg_values_t all_control_reg_values[] = { static struct { spinlock_t access_lock; -#if !MALI_USE_CSF - unsigned long prfcnt_base; -#endif /* !MALI_USE_CSF */ u32 *prfcnt_base_cpu; u32 time; @@ -474,11 +455,7 @@ static struct { u64 l2_present; u64 shader_present; -#if !MALI_USE_CSF - u64 jm_counters[KBASE_DUMMY_MODEL_COUNTER_PER_CORE]; -#else u64 cshw_counters[KBASE_DUMMY_MODEL_COUNTER_PER_CORE]; -#endif /* !MALI_USE_CSF */ u64 tiler_counters[KBASE_DUMMY_MODEL_COUNTER_PER_CORE]; u64 l2_counters[KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS * KBASE_DUMMY_MODEL_COUNTER_PER_CORE]; u64 shader_counters[KBASE_DUMMY_MODEL_MAX_SHADER_CORES * KBASE_DUMMY_MODEL_COUNTER_PER_CORE]; @@ -510,7 +487,6 @@ static u32 get_implementation_register(u32 reg, return 0; } -#if MALI_USE_CSF static u32 hctrl_get_implementation_register(u32 reg, const struct control_reg_values_t *const control_reg_values) @@ -531,7 +507,6 @@ hctrl_get_implementation_register(u32 reg, return 0; } -#endif void gpu_device_set_data(void *model, void *data) { @@ -553,7 +528,6 @@ static char *no_mali_gpu = CONFIG_MALI_VALHALL_NO_MALI_DEFAULT_GPU; module_param(no_mali_gpu, charp, 0000); MODULE_PARM_DESC(no_mali_gpu, "GPU to identify as"); -#if MALI_USE_CSF static u32 gpu_model_get_prfcnt_value(enum kbase_ipa_core_type core_type, u32 cnt_idx, bool is_low_word) { @@ -628,7 +602,6 @@ static u32 gpu_model_get_prfcnt_value(enum kbase_ipa_core_type core_type, u32 cn else return (value >> 32); } -#endif /* MALI_USE_CSF */ /** * gpu_model_clear_prfcnt_values_nolock - Clear performance counter values @@ -639,18 +612,13 @@ static u32 gpu_model_get_prfcnt_value(enum kbase_ipa_core_type core_type, u32 cn static void gpu_model_clear_prfcnt_values_nolock(void) { lockdep_assert_held(&performance_counters.access_lock); -#if !MALI_USE_CSF - memset(performance_counters.jm_counters, 0, sizeof(performance_counters.jm_counters)); -#else memset(performance_counters.cshw_counters, 0, sizeof(performance_counters.cshw_counters)); -#endif /* !MALI_USE_CSF */ memset(performance_counters.tiler_counters, 0, sizeof(performance_counters.tiler_counters)); memset(performance_counters.l2_counters, 0, sizeof(performance_counters.l2_counters)); memset(performance_counters.shader_counters, 0, sizeof(performance_counters.shader_counters)); } -#if MALI_USE_CSF void gpu_model_clear_prfcnt_values(void) { unsigned long flags; @@ -660,7 +628,6 @@ void gpu_model_clear_prfcnt_values(void) spin_unlock_irqrestore(&performance_counters.access_lock, flags); } KBASE_EXPORT_TEST_API(gpu_model_clear_prfcnt_values); -#endif /* MALI_USE_CSF */ /** * gpu_model_dump_prfcnt_blocks() - Dump performance counter values to buffer @@ -689,11 +656,9 @@ static void gpu_model_dump_prfcnt_blocks(u64 *values, u32 *out_index, u32 block_ for (block_idx = 0; block_idx < block_count; block_idx++) { /* only dump values if core is present */ if (!(blocks_present & (1U << block_idx))) { -#if MALI_USE_CSF /* if CSF dump zeroed out block */ memset(&prfcnt_base[*out_index], 0, KBASE_DUMMY_MODEL_BLOCK_SIZE); *out_index += KBASE_DUMMY_MODEL_VALUES_PER_BLOCK; -#endif /* MALI_USE_CSF */ continue; } @@ -724,13 +689,8 @@ static void gpu_model_dump_nolock(void) lockdep_assert_held(&performance_counters.access_lock); -#if !MALI_USE_CSF - gpu_model_dump_prfcnt_blocks(performance_counters.jm_counters, &index, 1, - performance_counters.prfcnt_en.fe, 0x1); -#else gpu_model_dump_prfcnt_blocks(performance_counters.cshw_counters, &index, 1, performance_counters.prfcnt_en.fe, 0x1); -#endif /* !MALI_USE_CSF */ gpu_model_dump_prfcnt_blocks(performance_counters.tiler_counters, &index, 1, performance_counters.prfcnt_en.tiler, DUMMY_IMPLEMENTATION_TILER_PRESENT); @@ -763,16 +723,6 @@ static void gpu_model_raise_irq(void *model, u32 irq) gpu_device_raise_irq(model, irq); } -#if !MALI_USE_CSF -static void midgard_model_dump_prfcnt(void) -{ - unsigned long flags; - - spin_lock_irqsave(&performance_counters.access_lock, flags); - gpu_model_dump_nolock(); - spin_unlock_irqrestore(&performance_counters.access_lock, flags); -} -#else void gpu_model_prfcnt_dump_request(u32 *sample_buf, struct gpu_model_prfcnt_en enable_maps) { unsigned long flags; @@ -796,7 +746,6 @@ void gpu_model_glb_request_job_irq(void *model) spin_unlock_irqrestore(&hw_error_status.access_lock, flags); gpu_model_raise_irq(model, MODEL_LINUX_JOB_IRQ); } -#endif /* !MALI_USE_CSF */ static void init_register_statuses(struct dummy_model_t *dummy) { @@ -830,117 +779,6 @@ static void update_register_statuses(struct dummy_model_t *dummy, u32 job_slot) if (hw_error_status.errors_mask & IS_A_JOB_ERROR) { if (job_slot == hw_error_status.current_job_slot) { -#if !MALI_USE_CSF - if (hw_error_status.js_status[job_slot] == 0) { - /* status reg is clean; it can be written */ - - switch (hw_error_status.errors_mask & IS_A_JOB_ERROR) { - case KBASE_JOB_INTERRUPTED: - hw_error_status.js_status[job_slot] = JS_STATUS_INTERRUPTED; - break; - - case KBASE_JOB_STOPPED: - hw_error_status.js_status[job_slot] = JS_STATUS_STOPPED; - break; - - case KBASE_JOB_TERMINATED: - hw_error_status.js_status[job_slot] = JS_STATUS_TERMINATED; - break; - - case KBASE_JOB_CONFIG_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_CONFIG_FAULT; - break; - - case KBASE_JOB_POWER_FAULT: - hw_error_status.js_status[job_slot] = JS_STATUS_POWER_FAULT; - break; - - case KBASE_JOB_READ_FAULT: - hw_error_status.js_status[job_slot] = JS_STATUS_READ_FAULT; - break; - - case KBASE_JOB_WRITE_FAULT: - hw_error_status.js_status[job_slot] = JS_STATUS_WRITE_FAULT; - break; - - case KBASE_JOB_AFFINITY_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_AFFINITY_FAULT; - break; - - case KBASE_JOB_BUS_FAULT: - hw_error_status.js_status[job_slot] = JS_STATUS_BUS_FAULT; - break; - - case KBASE_INSTR_INVALID_PC: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_INVALID_PC; - break; - - case KBASE_INSTR_INVALID_ENC: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_INVALID_ENC; - break; - - case KBASE_INSTR_TYPE_MISMATCH: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_TYPE_MISMATCH; - break; - - case KBASE_INSTR_OPERAND_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_OPERAND_FAULT; - break; - - case KBASE_INSTR_TLS_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_TLS_FAULT; - break; - - case KBASE_INSTR_BARRIER_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_BARRIER_FAULT; - break; - - case KBASE_INSTR_ALIGN_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_INSTR_ALIGN_FAULT; - break; - - case KBASE_DATA_INVALID_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_DATA_INVALID_FAULT; - break; - - case KBASE_TILE_RANGE_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_TILE_RANGE_FAULT; - break; - - case KBASE_ADDR_RANGE_FAULT: - hw_error_status.js_status[job_slot] = - JS_STATUS_ADDRESS_RANGE_FAULT; - break; - - case KBASE_OUT_OF_MEMORY: - hw_error_status.js_status[job_slot] = - JS_STATUS_OUT_OF_MEMORY; - break; - - case KBASE_UNKNOWN: - hw_error_status.js_status[job_slot] = JS_STATUS_UNKNOWN; - break; - - default: - model_error_log(KBASE_CORE, - "\nAtom Chain 0x%llx: Invalid Error Mask!", - hw_error_status.current_jc); - break; - } - } -#endif /* !MALI_USE_CSF */ - /* we set JOB_FAIL_ */ hw_error_status.job_irq_rawstat |= (dummy->slots[job_slot].job_complete_irq_asserted) @@ -1049,40 +887,6 @@ static void update_register_statuses(struct dummy_model_t *dummy, u32 job_slot) hw_error_status.errors_mask = 0; /*clear error mask */ } -#if !MALI_USE_CSF -static void update_job_irq_js_state(struct dummy_model_t *dummy, int mask) -{ - int i; - - lockdep_assert_held(&hw_error_status.access_lock); - pr_debug("%s", "Updating the JS_ACTIVE register"); - - for (i = 0; i < NUM_SLOTS; i++) { - int slot_active = dummy->slots[i].job_active; - int next_busy = dummy->slots[i].job_queued; - - if ((mask & (1 << i)) || (mask & (1 << (i + 16)))) { - /* clear the bits we're updating */ - dummy->job_irq_js_state &= ~((1 << (16 + i)) | (1 << i)); - if (hw_error_status.js_status[i]) { - dummy->job_irq_js_state |= next_busy << (i + 16); - if (mask & (1 << (i + 16))) { - /* clear job slot status */ - hw_error_status.js_status[i] = 0; - /* continue execution of jobchain */ - dummy->slots[i].job_active = dummy->slots[i].job_queued; - } - } else { - /* set bits if needed */ - dummy->job_irq_js_state |= - ((slot_active << i) | (next_busy << (i + 16))); - } - } - } - pr_debug("The new snapshot is 0x%08X\n", dummy->job_irq_js_state); -} -#endif /* !MALI_USE_CSF */ - /** * find_gpu_rev() - Append GPU HW revision to the GPU name, if needed. * @new_gpu_name: Caller-allocated string for the new GPU name. @@ -1121,6 +925,9 @@ static const struct control_reg_values_t *find_control_reg_values(const char *gp const struct control_reg_values_t *ret = NULL; char *gpu_with_rev = kmalloc(strlen(gpu) + GPU_REV_STR_LEN + 1, GFP_KERNEL); + if (gpu_with_rev == NULL) + return NULL; + /* Fix gpu name in case there are multiple entries for different HW versions. */ find_gpu_rev(gpu_with_rev, gpu); @@ -1164,10 +971,15 @@ void *midgard_model_create(struct kbase_device *kbdev) dummy->job_irq_js_state = 0; init_register_statuses(dummy); dummy->control_reg_values = find_control_reg_values(no_mali_gpu); + + if (dummy->control_reg_values == NULL) { + kfree(dummy); + return NULL; + } + dummy->arch_id = get_arch_id(dummy->control_reg_values->gpu_id); gpu_control_base_addr = GPU_CONTROL_BASE; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { performance_counters.l2_present = hctrl_get_implementation_register( GET_HOST_POWER_REG(HOST_POWER_L2_PRESENT_LO), @@ -1175,9 +987,7 @@ void *midgard_model_create(struct kbase_device *kbdev) performance_counters.shader_present = hctrl_get_implementation_register( GET_HOST_POWER_REG(HOST_POWER_SHADER_PRESENT_LO), dummy->control_reg_values); - } else -#endif /* MALI_USE_CSF */ - { + } else { performance_counters.l2_present = get_implementation_register( GET_GPU_CONTROL_REG(L2_PRESENT_LO), dummy->control_reg_values); performance_counters.shader_present = get_implementation_register( @@ -1299,73 +1109,6 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) spin_lock_irqsave(&hw_error_status.access_lock, flags); -#if !MALI_USE_CSF - if ((addr >= JOB_CONTROL_REG(JOB_SLOT0)) && (addr < (JOB_CONTROL_REG(JOB_SLOT15) + 0x80))) { - unsigned int slot_idx = (addr >> 7) & 0xf; - - KBASE_DEBUG_ASSERT(slot_idx < NUM_SLOTS); - if (addr == JOB_SLOT_REG(slot_idx, JS_HEAD_NEXT_LO)) { - hw_error_status.current_jc &= ~((u64)(0xFFFFFFFF)); - hw_error_status.current_jc |= (u64)value; - } - if (addr == JOB_SLOT_REG(slot_idx, JS_HEAD_NEXT_HI)) { - hw_error_status.current_jc &= (u64)0xFFFFFFFF; - hw_error_status.current_jc |= ((u64)value) << 32; - } - if (addr == JOB_SLOT_REG(slot_idx, JS_COMMAND_NEXT) && value == 1) { - pr_debug("%s", "start detected"); - KBASE_DEBUG_ASSERT(!dummy->slots[slot_idx].job_active || - !dummy->slots[slot_idx].job_queued); - if ((dummy->slots[slot_idx].job_active) || - (hw_error_status.job_irq_rawstat & (1 << (slot_idx + 16)))) { - pr_debug( - "~~~~~~~~~~~ Start: job slot is already active or there are IRQ pending ~~~~~~~~~"); - dummy->slots[slot_idx].job_queued = 1; - } else { - dummy->slots[slot_idx].job_active = 1; - } - } - - if (addr == JOB_SLOT_REG(slot_idx, JS_COMMAND_NEXT) && value == 0) - dummy->slots[slot_idx].job_queued = 0; - - if ((addr == JOB_SLOT_REG(slot_idx, JS_COMMAND)) && - (value == JS_COMMAND_SOFT_STOP || value == JS_COMMAND_HARD_STOP)) { - /*dummy->slots[slot_idx].job_active = 0; */ - hw_error_status.current_job_slot = slot_idx; - if (value == JS_COMMAND_SOFT_STOP) { - hw_error_status.errors_mask = KBASE_JOB_STOPPED; - } else { /*value == 3 */ - - if (dummy->slots[slot_idx].job_disabled != 0) { - pr_debug("enabling slot after HARD_STOP"); - dummy->slots[slot_idx].job_disabled = 0; - } - hw_error_status.errors_mask = KBASE_JOB_TERMINATED; - } - } - } else if (addr == JOB_CONTROL_REG(JOB_IRQ_CLEAR)) { - int i; - - for (i = 0; i < NUM_SLOTS; i++) { - if (value & ((1u << i) | (1u << (i + 16)))) - dummy->slots[i].job_complete_irq_asserted = 0; - /* hw_error_status.js_status[i] is cleared in - * update_job_irq_js_state - */ - } - pr_debug("%s", "job irq cleared"); - update_job_irq_js_state(dummy, value); - /*remove error condition for JOB */ - hw_error_status.job_irq_rawstat &= ~(value); - hw_error_status.job_irq_status &= ~(value); - } else if (addr == JOB_CONTROL_REG(JOB_IRQ_MASK)) { - int i; - - for (i = 0; i < NUM_SLOTS; i++) - dummy->slots[i].job_irq_mask = (value >> i) & 0x01; - pr_debug("job irq mask to value %x", value); -#else /* MALI_USE_CSF */ if (addr == JOB_CONTROL_REG(JOB_IRQ_CLEAR)) { pr_debug("%s", "job irq cleared"); @@ -1376,25 +1119,16 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) hw_error_status.job_irq_status |= value; } else if (addr == JOB_CONTROL_REG(JOB_IRQ_MASK)) { /* ignore JOB_IRQ_MASK as it is handled by CSFFW */ -#endif /* !MALI_USE_CSF */ } else if (addr == GET_GPU_CONTROL_REG(GPU_IRQ_MASK)) { pr_debug("GPU_IRQ_MASK set to 0x%x", value); -#if MALI_USE_CSF if (!dummy->kbdev->pm.backend.has_host_pwr_iface) { dummy->reset_completed_mask = (value >> 8) & 0x01; dummy->power_changed_mask = (value >> 9) & 0x03; } -#else - dummy->reset_completed_mask = (value >> 8) & 0x01; - dummy->power_changed_mask = (value >> 9) & 0x03; -#endif dummy->clean_caches_completed_irq_enabled = (value & (1u << 17)) != 0u; -#if MALI_USE_CSF dummy->flush_pa_range_completed_irq_enabled = (value & (1u << 20)) != 0u; -#endif } else if (addr == GET_GPU_CONTROL_REG(COHERENCY_ENABLE)) { dummy->coherency_enable = value; -#if MALI_USE_CSF } else if (addr == GET_HOST_POWER_REG(PWR_IRQ_MASK)) { pr_debug("PWR_IRQ_MASK set to 0x%x", value); dummy->power_changed_mask = (value & PWR_IRQ_POWER_CHANGED_SINGLE) | @@ -1531,28 +1265,12 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) if (value & (POWER_CHANGED_SINGLE | POWER_CHANGED_ALL)) dummy->power_changed = 0; } -#else - } else if (addr == GET_GPU_CONTROL_REG(GPU_IRQ_CLEAR)) { - if (value & RESET_COMPLETED) { - pr_debug("%s", "gpu RESET_COMPLETED irq cleared"); - dummy->reset_completed = 0; - } - if (value & (POWER_CHANGED_SINGLE | POWER_CHANGED_ALL)) - dummy->power_changed = 0; -#endif /* MALI_USE_CSF */ if (value & CLEAN_CACHES_COMPLETED) dummy->clean_caches_completed = false; -#if MALI_USE_CSF if (value & (1u << 20)) dummy->flush_pa_range_completed = false; -#endif /* MALI_USE_CSF */ - -#if !MALI_USE_CSF - if (value & PRFCNT_SAMPLE_COMPLETED) /* (1 << 16) */ - dummy->prfcnt_sample_completed = 0; -#endif /* !MALI_USE_CSF */ /*update error status */ hw_error_status.gpu_error_irq &= ~(value); @@ -1566,41 +1284,26 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) /* completed reset instantly */ dummy->reset_completed = 1; break; -#if MALI_USE_CSF case GPU_COMMAND_CACHE_CLN_INV_L2: case GPU_COMMAND_CACHE_CLN_INV_L2_LSC: case GPU_COMMAND_CACHE_CLN_INV_FULL: -#else - case GPU_COMMAND_CLEAN_CACHES: - case GPU_COMMAND_CLEAN_INV_CACHES: -#endif pr_debug("clean caches requested"); dummy->clean_caches_completed = true; break; -#if MALI_USE_CSF case GPU_COMMAND_FLUSH_PA_RANGE_CLN_INV_L2: case GPU_COMMAND_FLUSH_PA_RANGE_CLN_INV_L2_LSC: case GPU_COMMAND_FLUSH_PA_RANGE_CLN_INV_FULL: pr_debug("pa range flush requested"); dummy->flush_pa_range_completed = true; break; -#endif /* MALI_USE_CSF */ -#if !MALI_USE_CSF - case GPU_COMMAND_PRFCNT_SAMPLE: - midgard_model_dump_prfcnt(); - dummy->prfcnt_sample_completed = 1; -#endif /* !MALI_USE_CSF */ default: break; } -#if MALI_USE_CSF } else if (addr >= GET_GPU_CONTROL_REG(GPU_COMMAND_ARG0_LO) && addr <= GET_GPU_CONTROL_REG(GPU_COMMAND_ARG1_HI)) { /* Writes ignored */ -#endif } else if (addr == GET_GPU_CONTROL_REG(L2_CONFIG)) { dummy->l2_config = value; -#if MALI_USE_CSF } else if (addr >= CSF_HW_DOORBELL_PAGE_OFFSET && addr < CSF_HW_DOORBELL_PAGE_OFFSET + (dummy->kbdev->csf.num_doorbells * CSF_HW_DOORBELL_PAGE_SIZE)) { @@ -1650,7 +1353,6 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) } } else if (addr == GPU_GOV_CORE_MASK_OFFSET) { dummy->gov_core_mask = value; -#endif /* MALI_USE_CSF */ } else if (addr == MMU_CONTROL_REG(MMU_IRQ_MASK)) { hw_error_status.mmu_irq_mask = value; } else if (addr == MMU_CONTROL_REG(MMU_IRQ_CLEAR)) { @@ -1659,6 +1361,9 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) (addr <= MMU_STAGE1_REG(MMU_AS_REG(15, AS_STATUS)))) { u32 mem_addr_space = (addr - MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) >> MMU_STAGE1_AS_SHIFT; + if (WARN_ON(mem_addr_space >= NUM_MMU_AS)) { + return; + } switch (addr & 0x3F) { case AS_COMMAND: @@ -1737,32 +1442,6 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) } } else { switch (addr) { -#if !MALI_USE_CSF - case PRFCNT_BASE_LO: - performance_counters.prfcnt_base = - HI_MASK(performance_counters.prfcnt_base) | value; - performance_counters.prfcnt_base_cpu = - (u32 *)(uintptr_t)performance_counters.prfcnt_base; - break; - case PRFCNT_BASE_HI: - performance_counters.prfcnt_base = - LO_MASK(performance_counters.prfcnt_base) | (((u64)value) << 32); - performance_counters.prfcnt_base_cpu = - (u32 *)(uintptr_t)performance_counters.prfcnt_base; - break; - case PRFCNT_JM_EN: - performance_counters.prfcnt_en.fe = value; - break; - case PRFCNT_SHADER_EN: - performance_counters.prfcnt_en.shader = value; - break; - case PRFCNT_TILER_EN: - performance_counters.prfcnt_en.tiler = value; - break; - case PRFCNT_MMU_L2_EN: - performance_counters.prfcnt_en.l2 = value; - break; -#endif /* !MALI_USE_CSF */ case TILER_PWRON_LO: dummy->domain_power_on[INDEX_TILER] |= value & DUMMY_IMPLEMENTATION_TILER_PRESENT; @@ -1813,13 +1492,8 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) case PWR_KEY: case PWR_OVERRIDE0: case PWR_OVERRIDE1: -#if MALI_USE_CSF case SHADER_PWRFEATURES: case CSF_CONFIG: -#else /* !MALI_USE_CSF */ - case JM_CONFIG: - case PRFCNT_CONFIG: -#endif /* MALI_USE_CSF */ case SHADER_CONFIG: case TILER_CONFIG: case L2_MMU_CONFIG: @@ -1847,15 +1521,7 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) spin_lock_irqsave(&hw_error_status.access_lock, flags); *value = 0; /* 0 by default */ -#if !MALI_USE_CSF - if (addr == JOB_CONTROL_REG(JOB_IRQ_JS_STATE)) { - pr_debug("%s", "JS_ACTIVE being read"); - - *value = dummy->job_irq_js_state; - } else if (addr == GET_GPU_CONTROL_REG(GPU_ID)) { -#else /* !MALI_USE_CSF */ if (addr == GET_GPU_CONTROL_REG(GPU_ID)) { -#endif /* !MALI_USE_CSF */ *value = dummy->control_reg_values->gpu_id & U32_MAX; } else if (addr == JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)) { *value = hw_error_status.job_irq_rawstat; @@ -1863,42 +1529,21 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) } else if (addr == JOB_CONTROL_REG(JOB_IRQ_STATUS)) { *value = hw_error_status.job_irq_status; pr_debug("JS_IRQ_STATUS being read %x", *value); -#if !MALI_USE_CSF - } else if (addr == JOB_CONTROL_REG(JOB_IRQ_MASK)) { - int i; - - *value = 0; - for (i = 0; i < NUM_SLOTS; i++) - *value |= dummy->slots[i].job_irq_mask << i; - pr_debug("JS_IRQ_MASK being read %x", *value); -#else /* !MALI_USE_CSF */ } else if (addr == JOB_CONTROL_REG(JOB_IRQ_MASK)) { /* ignore JOB_IRQ_MASK as it is handled by CSFFW */ -#endif /* !MALI_USE_CSF */ } else if (addr == GET_GPU_CONTROL_REG(GPU_IRQ_MASK)) { *value = (dummy->reset_completed_mask << 8) | ((dummy->clean_caches_completed_irq_enabled ? 1u : 0u) << 17) | -#if MALI_USE_CSF ((dummy->flush_pa_range_completed_irq_enabled ? 1u : 0u) << 20) | -#endif (dummy->power_changed_mask << 9) | (1u << 7) | 1u; pr_debug("GPU_IRQ_MASK read %x", *value); } else if (addr == GET_GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) { *value = ((dummy->clean_caches_completed ? 1u : 0u) << 17) | -#if MALI_USE_CSF ((dummy->flush_pa_range_completed ? 1u : 0u) << 20) | -#else - (dummy->prfcnt_sample_completed ? PRFCNT_SAMPLE_COMPLETED : 0) | -#endif hw_error_status.gpu_error_irq; -#if MALI_USE_CSF if (!dummy->kbdev->pm.backend.has_host_pwr_iface) *value |= (dummy->power_changed << 9) | (dummy->power_changed << 10) | (dummy->reset_completed << 8); -#else - *value |= (dummy->power_changed << 9) | (dummy->power_changed << 10) | - (dummy->reset_completed << 8); -#endif pr_debug("GPU_IRQ_RAWSTAT read %x", *value); } else if (addr == GET_GPU_CONTROL_REG(GPU_IRQ_STATUS)) { @@ -1907,40 +1552,25 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) 1u : 0u) << 17) | -#if MALI_USE_CSF (((dummy->flush_pa_range_completed && dummy->flush_pa_range_completed_irq_enabled) ? 1u : 0u) << 20) | -#else - (dummy->prfcnt_sample_completed ? PRFCNT_SAMPLE_COMPLETED : 0) | -#endif hw_error_status.gpu_error_irq; -#if MALI_USE_CSF if (!dummy->kbdev->pm.backend.has_host_pwr_iface) *value |= ((dummy->power_changed && (dummy->power_changed_mask & 0x1)) << 9) | ((dummy->power_changed && (dummy->power_changed_mask & 0x2)) << 10) | ((dummy->reset_completed & dummy->reset_completed_mask) << 8); -#else - *value |= ((dummy->power_changed && (dummy->power_changed_mask & 0x1)) << 9) | - ((dummy->power_changed && (dummy->power_changed_mask & 0x2)) << 10) | - ((dummy->reset_completed & dummy->reset_completed_mask) << 8); -#endif pr_debug("GPU_IRQ_STAT read %x", *value); } else if (addr == GET_GPU_CONTROL_REG(GPU_STATUS)) { *value = 0; -#if !MALI_USE_CSF - } else if (addr == GET_GPU_CONTROL_REG(LATEST_FLUSH)) { - *value = 0; -#endif } else if (addr == GET_GPU_CONTROL_REG(GPU_FAULTSTATUS)) { *value = hw_error_status.gpu_fault_status; } else if (addr == GET_GPU_CONTROL_REG(L2_CONFIG)) { *value = dummy->l2_config; -#if MALI_USE_CSF } else if ((addr >= GET_GPU_CONTROL_REG(SYSC_ALLOC0)) && (addr < GET_GPU_CONTROL_REG(SYSC_ALLOC(SYSC_ALLOC_COUNT)))) { u32 alloc_reg = (addr - GET_GPU_CONTROL_REG(SYSC_ALLOC0)) >> 2; @@ -2042,8 +1672,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) addr == GET_HOST_POWER_REG(HOST_POWER_NEURAL_PWRTRANS_LO) || addr == GET_HOST_POWER_REG(HOST_POWER_NEURAL_PWRTRANS_HI) || - addr == GET_HOST_POWER_REG(HOST_POWER_L2_PWRACTIVE_LO) || - addr == GET_HOST_POWER_REG(HOST_POWER_L2_PWRACTIVE_HI) || addr == GET_HOST_POWER_REG(HOST_POWER_TILER_PWRACTIVE_LO) || addr == GET_HOST_POWER_REG(HOST_POWER_TILER_PWRACTIVE_HI) || addr == GET_HOST_POWER_REG(HOST_POWER_SHADER_PWRACTIVE_LO) || @@ -2066,7 +1694,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) "Dummy model register access: Reading unknown control reg 0x%x\n", addr); } -#endif } else if ((addr >= GET_GPU_CONTROL_REG(SHADER_PRESENT_LO)) && (addr <= GET_GPU_CONTROL_REG(L2_MMU_CONFIG))) { if (addr == GET_GPU_CONTROL_REG(SHADER_PRESENT_LO) || @@ -2106,8 +1733,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) addr == GET_GPU_CONTROL_REG(SHADER_PWRTRANS_HI) || addr == GET_GPU_CONTROL_REG(STACK_PWRTRANS_LO) || addr == GET_GPU_CONTROL_REG(STACK_PWRTRANS_HI) || - addr == GET_GPU_CONTROL_REG(L2_PWRACTIVE_LO) || - addr == GET_GPU_CONTROL_REG(L2_PWRACTIVE_HI) || addr == GET_GPU_CONTROL_REG(TILER_PWRACTIVE_LO) || addr == GET_GPU_CONTROL_REG(TILER_PWRACTIVE_HI) || addr == GET_GPU_CONTROL_REG(SHADER_PWRACTIVE_LO) || @@ -2131,51 +1756,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) addr); } -#if !MALI_USE_CSF - } else if ((addr >= JOB_CONTROL_REG(JOB_SLOT0)) && - (addr < (JOB_CONTROL_REG(JOB_SLOT15) + 0x80))) { - int slot_idx = (addr >> 7) & 0xf; - int sub_reg = addr & 0x7F; - - KBASE_DEBUG_ASSERT(slot_idx < NUM_SLOTS); - switch (sub_reg) { - case JS_HEAD_NEXT_LO: - *value = (u32)((hw_error_status.current_jc) & 0xFFFFFFFF); - break; - case JS_HEAD_NEXT_HI: - *value = (u32)(hw_error_status.current_jc >> 32); - break; - case JS_STATUS: - if (hw_error_status.js_status[slot_idx]) - *value = hw_error_status.js_status[slot_idx]; - else /* 0x08 means active, 0x00 idle */ - *value = (dummy->slots[slot_idx].job_active) << 3; - break; - case JS_COMMAND_NEXT: - *value = dummy->slots[slot_idx].job_queued; - break; - - /** - * The dummy model does not implement these registers - * avoid printing error messages - */ - case JS_HEAD_HI: - case JS_HEAD_LO: - case JS_TAIL_HI: - case JS_TAIL_LO: - case JS_FLUSH_ID_NEXT: - break; - - default: - model_error_log( - KBASE_CORE, - "Dummy model register access: unknown job slot reg 0x%02X being read\n", - sub_reg); - break; - } - } else if (addr == GET_GPU_CONTROL_REG(JS_PRESENT)) { - *value = 0x7; -#endif /* !MALI_USE_CSF */ } else if (addr == GET_GPU_CONTROL_REG(AS_PRESENT)) { *value = dummy->control_reg_values->as_present; } else if (addr >= GET_GPU_CONTROL_REG(TEXTURE_FEATURES_0) && @@ -2188,27 +1768,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) *value = 0x9f81ffff; else if (addr == GET_GPU_CONTROL_REG(TEXTURE_FEATURES_3)) *value = 0; -#if !MALI_USE_CSF - } else if (addr >= GPU_CONTROL_REG(JS0_FEATURES) && - addr <= GPU_CONTROL_REG(JS15_FEATURES)) { - switch (addr) { - case GPU_CONTROL_REG(JS0_FEATURES): - *value = 0x20e; - break; - - case GPU_CONTROL_REG(JS1_FEATURES): - *value = 0x1fe; - break; - - case GPU_CONTROL_REG(JS2_FEATURES): - *value = 0x7e; - break; - - default: - *value = 0; - break; - } -#endif /* !MALI_USE_CSF */ } else if (addr >= GET_GPU_CONTROL_REG(L2_FEATURES) && addr <= GET_GPU_CONTROL_REG(MMU_FEATURES)) { if (addr == GET_GPU_CONTROL_REG(L2_FEATURES)) @@ -2242,8 +1801,11 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) } else if ((addr >= MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) && (addr <= MMU_STAGE1_REG(MMU_AS_REG(15, AS_STATUS)))) { - u32 mem_addr_space = (addr - MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) >> - MMU_STAGE1_AS_SHIFT; + u32 mem_addr_space; + + mem_addr_space = (addr - MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) >> + MMU_STAGE1_AS_SHIFT; + switch (addr & 0x3F) { case AS_TRANSTAB_LO: *value = (u32)(hw_error_status.as_transtab[mem_addr_space] & 0xffffffff); @@ -2289,7 +1851,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) *value = hw_error_status.mmu_irq_rawstat; } else if (addr == MMU_CONTROL_REG(MMU_IRQ_STATUS)) { *value = hw_error_status.mmu_irq_mask & hw_error_status.mmu_irq_rawstat; -#if MALI_USE_CSF } else if (addr == IPA_CONTROL_REG(STATUS) || addr == IPA_CONTROL_REG(STATUS) + GPU_GOV_IPA_CONTROL_OFFSET) { *value = (ipa_control_timer_enabled << 31); @@ -2382,7 +1943,6 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) *value = gpu_model_get_prfcnt_value(KBASE_IPA_CORE_TYPE_NEURAL, counter_index, is_low_word); -#endif /* MALI_USE_CSF */ } else if (addr == GET_GPU_CONTROL_REG(GPU_FEATURES_LO)) { *value = dummy->control_reg_values->gpu_features_lo; } else if (addr == GET_GPU_CONTROL_REG(GPU_FEATURES_HI)) { @@ -2473,13 +2033,8 @@ int gpu_model_set_dummy_prfcnt_user_sample(u32 __user *data, u32 size) } spin_lock_irqsave(&performance_counters.access_lock, flags); -#if !MALI_USE_CSF - offset = set_user_sample_core_type(performance_counters.jm_counters, user_data, offset, - size, 1); -#else offset = set_user_sample_core_type(performance_counters.cshw_counters, user_data, offset, size, 1); -#endif /* !MALI_USE_CSF */ offset = set_user_sample_core_type(performance_counters.tiler_counters, user_data, offset, size, hweight64(DUMMY_IMPLEMENTATION_TILER_PRESENT)); offset = set_user_sample_core_type(performance_counters.l2_counters, user_data, offset, @@ -2499,13 +2054,8 @@ void gpu_model_set_dummy_prfcnt_kernel_sample(u64 *data, u32 size) u32 offset = 0; spin_lock_irqsave(&performance_counters.access_lock, flags); -#if !MALI_USE_CSF - offset = set_kernel_sample_core_type(performance_counters.jm_counters, data, offset, size, - 1); -#else offset = set_kernel_sample_core_type(performance_counters.cshw_counters, data, offset, size, 1); -#endif /* !MALI_USE_CSF */ offset = set_kernel_sample_core_type(performance_counters.tiler_counters, data, offset, size, hweight64(DUMMY_IMPLEMENTATION_TILER_PRESENT)); offset = set_kernel_sample_core_type(performance_counters.l2_counters, data, offset, size, diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.h index 115d31a6f560..c57c3020c51a 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_model_dummy.h @@ -202,7 +202,6 @@ void gpu_model_set_dummy_prfcnt_cores(struct kbase_device *kbdev, u64 l2_present /* Clear the counter values array maintained by the dummy model */ void gpu_model_clear_prfcnt_values(void); -#if MALI_USE_CSF /** * gpu_model_prfcnt_dump_request() - Request performance counter sample dump. * @sample_buf: Pointer to KBASE_DUMMY_MODEL_MAX_VALUES_PER_SAMPLE sized array @@ -217,7 +216,6 @@ void gpu_model_prfcnt_dump_request(uint32_t *sample_buf, struct gpu_model_prfcnt * @model: Model pointer returned by midgard_model_create(). */ void gpu_model_glb_request_job_irq(void *model); -#endif /* MALI_USE_CSF */ extern struct error_status_t hw_error_status; diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_always_on.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_always_on.c index 37c35ee9bd32..12bb7448f619 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_always_on.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_always_on.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -67,9 +67,7 @@ const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { always_on_get_core_active, /* get_core_active */ NULL, /* handle_event */ KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ -#if MALI_USE_CSF ALWAYS_ON_PM_SCHED_FLAGS, /* pm_sched_flags */ -#endif }; KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_backend.c index a61809dd2fda..28665a8f1ee1 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_backend.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_backend.c @@ -29,20 +29,13 @@ #include #include -#if !MALI_USE_CSF -#include -#include -#include -#else #include #include #include #include -#endif /* !MALI_USE_CSF */ #include #include #include -#include #include @@ -99,9 +92,6 @@ void kbase_pm_register_access_enable(struct kbase_device *kbdev) if (callbacks) callbacks->power_on_callback(kbdev); - if (WARN_ON(kbase_io_is_aw_removed(kbdev))) - dev_err(kbdev->dev, "Attempting to power on while GPU lost\n"); - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_GPU_OFF); } @@ -124,6 +114,7 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) KBASE_DEBUG_ASSERT(kbdev != NULL); mutex_init(&kbdev->pm.lock); + kbdev->pm.runtime_suspend_result = 0; kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait", WQ_HIGHPRI | WQ_UNBOUND, 1); @@ -133,22 +124,17 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, kbase_pm_gpu_poweroff_wait_wq); kbdev->pm.backend.ca_cores_enabled = ~0ull; - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + kbdev->pm.backend.ca_gov_cores_enabled = ~0ull; + init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); -#if !MALI_USE_CSF - /* Initialise the metrics subsystem */ - ret = kbasep_pm_metrics_init(kbdev); - if (ret) - return ret; -#else mutex_init(&kbdev->pm.backend.policy_change_lock); kbdev->pm.backend.policy_change_clamp_state_to_off = false; /* Due to dependency on kbase_ipa_control, the metrics subsystem can't * be initialized here. */ CSTD_UNUSED(ret); -#endif init_waitqueue_head(&kbdev->pm.backend.reset_done_wait); kbdev->pm.backend.reset_done = false; @@ -162,12 +148,10 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) init_waitqueue_head(&kbdev->pm.backend.poweroff_wait); -#if MALI_USE_CSF /* Select the power interface that the GPU is using. */ kbdev->pm.backend.has_host_pwr_iface = kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_POWER_CONTROL); kbdev->pm.backend.pwr_cntl_delegated = false; -#endif if (kbase_pm_ca_init(kbdev) != 0) goto workq_fail; @@ -181,24 +165,27 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->pm.backend.hwcnt_disable_work, kbase_pm_hwcnt_disable_worker); kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - kbdev->pm.backend.gpu_sleep_allowed = 0; - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GPU_SLEEP) && - !kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TURSEHW_1997) && - kbdev->pm.backend.callback_power_runtime_gpu_active && - kbdev->pm.backend.callback_power_runtime_gpu_idle) - set_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); + if (IS_ENABLED(CONFIG_PM)) { + kbdev->pm.backend.gpu_sleep_allowed = 0; + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GPU_SLEEP) && + !kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TURSEHW_1997) && + kbdev->pm.backend.callback_power_runtime_gpu_active && + kbdev->pm.backend.callback_power_runtime_gpu_idle) + set_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); - kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa = - kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TITANHW_2938) && - test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); + kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa = + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TITANHW_2938) && + test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, + &kbdev->pm.backend.gpu_sleep_allowed); - /* FW Sleep-on-Idle is only available in certain architecture revisions */ - if ((kbdev->gpu_props.gpu_id.arch_major > 11) || - ((kbdev->gpu_props.gpu_id.arch_major == 11) && - (kbdev->gpu_props.gpu_id.arch_minor >= 8) && (kbdev->gpu_props.gpu_id.arch_rev >= 10))) - set_bit(KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE, &kbdev->pm.backend.gpu_sleep_allowed); -#endif + /* FW Sleep-on-Idle is only available in certain architecture revisions */ + if ((kbdev->gpu_props.gpu_id.arch_major > 11) || + ((kbdev->gpu_props.gpu_id.arch_major == 11) && + (kbdev->gpu_props.gpu_id.arch_minor >= 8) && + (kbdev->gpu_props.gpu_id.arch_rev >= 10))) + set_bit(KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE, + &kbdev->pm.backend.gpu_sleep_allowed); + } if (IS_ENABLED(CONFIG_MALI_VALHALL_HW_ERRATA_1485982_NOT_AFFECTED)) return 0; @@ -225,9 +212,6 @@ pm_state_machine_fail: kbase_pm_policy_term(kbdev); kbase_pm_ca_term(kbdev); workq_fail: -#if !MALI_USE_CSF - kbasep_pm_metrics_term(kbdev); -#endif return -EINVAL; } @@ -259,7 +243,6 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) */ } -#if MALI_USE_CSF static bool wait_cond_mmu_fault_handling_in_gpu_poweroff_wait_wq(struct kbase_device *kbdev, int faults_pending) { @@ -272,7 +255,6 @@ static bool wait_cond_mmu_fault_handling_in_gpu_poweroff_wait_wq(struct kbase_de return cond; } -#endif /** * wait_for_mmu_fault_handling_in_gpu_poweroff_wait() - Wait for pending MMU @@ -288,7 +270,6 @@ static bool wait_cond_mmu_fault_handling_in_gpu_poweroff_wait_wq(struct kbase_de */ static void wait_for_mmu_fault_handling_in_gpu_poweroff_wait(struct kbase_device *kbdev) { -#if MALI_USE_CSF bool reset_triggered = false; int ret = 0; @@ -318,19 +299,12 @@ static void wait_for_mmu_fault_handling_in_gpu_poweroff_wait(struct kbase_device } } while (ret); kbdev->pm.backend.waiting_for_mmu_fault_handling = false; -#else - kbase_pm_unlock(kbdev); - kbase_flush_mmu_wqs(kbdev); - kbase_pm_lock(kbdev); -#endif } static void pm_handle_power_off(struct kbase_device *kbdev) { struct kbase_pm_backend_data *backend = &kbdev->pm.backend; -#if MALI_USE_CSF enum kbase_mcu_state mcu_state; -#endif unsigned long flags; lockdep_assert_held(&kbdev->pm.lock); @@ -339,28 +313,24 @@ static void pm_handle_power_off(struct kbase_device *kbdev) return; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - if (kbdev->pm.backend.gpu_wakeup_override) { + + if (IS_ENABLED(CONFIG_PM) && kbdev->pm.backend.gpu_wakeup_override) { spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return; } -#endif + WARN_ON(backend->shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF || backend->l2_state != KBASE_L2_OFF); -#if MALI_USE_CSF mcu_state = backend->mcu_state; WARN_ON(!kbase_pm_is_mcu_inactive(kbdev, mcu_state)); -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - if (backend->callback_power_runtime_gpu_idle) { + if (IS_ENABLED(CONFIG_PM) && backend->callback_power_runtime_gpu_idle) { WARN_ON(backend->gpu_idled); backend->callback_power_runtime_gpu_idle(kbdev); backend->gpu_idled = true; return; } -#endif /* Disable interrupts and turn the clock off */ if (unlikely(!kbase_pm_clock_off(kbdev))) { @@ -398,21 +368,12 @@ void kbase_pm_handle_gpu_poweroff_wait_work(struct kbase_device *kbdev) KBASE_KTRACE_ADD(kbdev, PM_POWEROFF_WAIT_WQ, NULL, 0); -#if !MALI_USE_CSF - /* Wait for power transitions to complete. We do this with no locks held - * so that we don't deadlock with any pending workqueues. - */ - kbase_pm_wait_for_desired_state(kbdev); -#endif - kbase_pm_lock(kbdev); -#if MALI_USE_CSF spin_lock_irqsave(&kbdev->hwaccess_lock, flags); if (pm->backend.invoke_poweroff_wait_wq_when_l2_off || !pm->backend.poweroff_wait_in_progress) goto wakeup_exit; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#endif pm_handle_power_off(kbdev); @@ -421,18 +382,11 @@ void kbase_pm_handle_gpu_poweroff_wait_work(struct kbase_device *kbdev) if (backend->poweron_required) { backend->poweron_required = false; kbdev->pm.backend.l2_desired = true; -#if MALI_USE_CSF kbdev->pm.backend.mcu_desired = true; -#endif kbase_pm_update_state(kbdev); kbase_pm_update_cores_state_nolock(kbdev); -#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -#endif /* !MALI_USE_CSF */ } -#if MALI_USE_CSF wakeup_exit: -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); kbase_pm_unlock(kbdev); wake_up(&kbdev->pm.backend.poweroff_wait); @@ -589,28 +543,20 @@ static void kbase_pm_hwcnt_disable_worker(struct work_struct *data) */ backend->hwcnt_disabled = true; kbase_pm_update_state(kbdev); -#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -#endif /* !MALI_USE_CSF */ } else { /* PM state was updated while we were doing the disable, * so we need to undo the disable we just performed. */ -#if MALI_USE_CSF unsigned long lock_flags; kbase_csf_scheduler_spin_lock(kbdev, &lock_flags); -#endif kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -#if MALI_USE_CSF kbase_csf_scheduler_spin_unlock(kbdev, lock_flags); -#endif } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) /** * kbase_pm_do_poweroff_sync - Do the synchronous power down of GPU * @@ -679,7 +625,6 @@ out: kbase_pm_unlock(kbdev); return ret; } -#endif void kbase_pm_do_poweroff(struct kbase_device *kbdev) { @@ -695,12 +640,7 @@ void kbase_pm_do_poweroff(struct kbase_device *kbdev) if (kbdev->pm.backend.poweroff_wait_in_progress) goto unlock_hwaccess; -#if MALI_USE_CSF kbdev->pm.backend.mcu_desired = false; -#else - /* Force all cores off */ - kbdev->pm.backend.shaders_desired = false; -#endif kbdev->pm.backend.l2_desired = false; kbdev->pm.backend.poweroff_wait_in_progress = true; @@ -744,7 +684,10 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, unsigned int flags) kbase_pm_unlock(kbdev); return ret; } -#if MALI_USE_CSF + + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + kbdev->pm.sysfs_gov_core_mask = kbdev->gpu_props.shader_present; + kbdev->pm.debug_core_mask = kbdev->gpu_props.shader_present; spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); /* Set the initial value for 'shaders_avail'. It would be later @@ -755,25 +698,19 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, unsigned int flags) */ kbdev->pm.backend.shaders_avail = kbase_pm_ca_get_core_mask(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -#else - kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] = - kbdev->pm.debug_core_mask[1] = kbdev->pm.debug_core_mask[2] = - kbdev->gpu_props.shader_present; -#endif /* Pretend the GPU is active to prevent a power policy turning the GPU * cores off */ kbdev->pm.active_count = 1; -#if MALI_USE_CSF && KBASE_PM_RUNTIME - if (kbdev->pm.backend.callback_power_runtime_gpu_active) { + + if (IS_ENABLED(CONFIG_PM) && kbdev->pm.backend.callback_power_runtime_gpu_active) { /* Take the RPM reference count to match with the internal * PM reference count */ kbdev->pm.backend.callback_power_runtime_gpu_active(kbdev); WARN_ON(kbdev->pm.backend.gpu_idled); } -#endif spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, irq_flags); /* Ensure cycle counter is off */ @@ -790,12 +727,10 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, unsigned int flags) kbdev->pm.backend.gpu_ready = true; /* Turn on the GPU and any cores needed by the policy */ -#if MALI_USE_CSF /* Turn on the L2 caches, needed for firmware boot */ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); kbdev->pm.backend.l2_desired = true; spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -#endif kbase_pm_do_poweron(kbdev, false); kbase_pm_unlock(kbdev); @@ -806,15 +741,14 @@ void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) { KBASE_DEBUG_ASSERT(kbdev != NULL); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - WARN_ON(kbase_pm_do_poweroff_sync(kbdev)); -#else - mutex_lock(&kbdev->pm.lock); - kbase_pm_do_poweroff(kbdev); - mutex_unlock(&kbdev->pm.lock); - - kbase_pm_wait_for_poweroff_work_complete(kbdev); -#endif + if (IS_ENABLED(CONFIG_PM)) + WARN_ON(kbase_pm_do_poweroff_sync(kbdev)); + else { + mutex_lock(&kbdev->pm.lock); + kbase_pm_do_poweroff(kbdev); + mutex_unlock(&kbdev->pm.lock); + kbase_pm_wait_for_poweroff_work_complete(kbdev); + } } KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); @@ -827,17 +761,12 @@ void kbase_hwaccess_pm_term(struct kbase_device *kbdev) cancel_work_sync(&kbdev->pm.backend.hwcnt_disable_work); - if (kbdev->pm.backend.hwcnt_disabled) { + if (kbdev->pm.backend.hwcnt_disabled && kbdev->csf.firmware_hctl_core_pwr) { unsigned long flags; -#if MALI_USE_CSF + kbase_csf_scheduler_spin_lock(kbdev, &flags); kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); kbase_csf_scheduler_spin_unlock(kbdev, flags); -#else - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#endif } /* Free any resources the policy allocated */ @@ -845,16 +774,11 @@ void kbase_hwaccess_pm_term(struct kbase_device *kbdev) kbase_pm_policy_term(kbdev); kbase_pm_ca_term(kbdev); -#if !MALI_USE_CSF - /* Shut down the metrics subsystem */ - kbasep_pm_metrics_term(kbdev); -#else if (WARN_ON(mutex_is_locked(&kbdev->pm.backend.policy_change_lock))) { mutex_lock(&kbdev->pm.backend.policy_change_lock); mutex_unlock(&kbdev->pm.backend.policy_change_lock); } mutex_destroy(&kbdev->pm.backend.policy_change_lock); -#endif destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); } @@ -866,48 +790,23 @@ void kbase_pm_power_changed(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_pm_update_state(kbdev); -#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -#endif /* !MALI_USE_CSF */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -#if MALI_USE_CSF void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask) { lockdep_assert_held(&kbdev->hwaccess_lock); lockdep_assert_held(&kbdev->pm.lock); - kbdev->pm.debug_core_mask = new_core_mask; - kbase_pm_update_dynamic_cores_onoff(kbdev); + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + kbdev->pm.sysfs_gov_core_mask = new_core_mask; + kbase_pm_ca_set_gov_core_mask_nolock(kbdev, SYSFS_COREMASK, new_core_mask); + } else { + kbdev->pm.debug_core_mask = new_core_mask; + kbase_pm_update_dynamic_cores_onoff(kbdev); + } } KBASE_EXPORT_TEST_API(kbase_pm_set_debug_core_mask); -#else -void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 *new_core_mask, - size_t new_core_mask_size) -{ - size_t i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - lockdep_assert_held(&kbdev->pm.lock); - - if (kbase_dummy_job_wa_enabled(kbdev)) { - dev_warn_once( - kbdev->dev, - "Change of core mask not supported for slot 0 as dummy job WA is enabled"); - new_core_mask[0] = kbdev->pm.debug_core_mask[0]; - } - - kbdev->pm.debug_core_mask_all = 0; - for (i = 0; i < new_core_mask_size; i++) { - kbdev->pm.debug_core_mask[i] = new_core_mask[i]; - kbdev->pm.debug_core_mask_all |= new_core_mask[i]; - } - - kbase_pm_update_dynamic_cores_onoff(kbdev); -} -#endif /* MALI_USE_CSF */ void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) { @@ -923,35 +822,23 @@ int kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) { int ret = 0; -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - ret = kbase_pm_do_poweroff_sync(kbdev); - if (ret) - return ret; -#else - /* Force power off the GPU and all cores (regardless of policy), only - * after the PM active count reaches zero (otherwise, we risk turning it - * off prematurely) - */ - kbase_pm_lock(kbdev); + if (IS_ENABLED(CONFIG_PM)) { + ret = kbase_pm_do_poweroff_sync(kbdev); + if (ret) + return ret; + } else { + /* Force power off the GPU and all cores (regardless of policy), only + * after the PM active count reaches zero (otherwise, we risk turning it + * off prematurely) + */ + kbase_pm_lock(kbdev); + kbase_pm_do_poweroff(kbdev); + kbase_pm_unlock(kbdev); - kbase_pm_do_poweroff(kbdev); - -#if !MALI_USE_CSF - kbase_backend_timer_suspend(kbdev); -#endif /* !MALI_USE_CSF */ - - kbase_pm_unlock(kbdev); - - ret = kbase_pm_wait_for_poweroff_work_complete(kbdev); - if (ret) { -#if !MALI_USE_CSF - mutex_lock(&kbdev->js_data.runpool_mutex); - kbase_backend_timer_resume(kbdev); - mutex_unlock(&kbdev->js_data.runpool_mutex); -#endif /* !MALI_USE_CSF */ - return ret; + ret = kbase_pm_wait_for_poweroff_work_complete(kbdev); + if (ret) + return ret; } -#endif WARN_ON(kbase_io_is_gpu_powered(kbdev)); WARN_ON(atomic_read(&kbdev->faults_pending)); @@ -976,10 +863,6 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) } kbase_pm_do_poweron(kbdev, true); -#if !MALI_USE_CSF - kbase_backend_timer_resume(kbdev); -#endif /* !MALI_USE_CSF */ - kbase_pm_unlock(kbdev); } @@ -995,24 +878,23 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) return; } -#if MALI_USE_CSF - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (kbase_io_is_gpu_powered(kbdev) && !kbase_io_is_aw_removed(kbdev)) { + if (kbase_io_is_aw_removed(kbdev)) { unsigned long flags_sched; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* GPU is no longer mapped to VM. So no interrupts will * be received and Mali registers have been replaced by * dummy RAM */ - WARN(kbase_io_has_gpu(kbdev), "GPU is still available after GPU lost event\n"); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + kbase_csf_scheduler_spin_lock(kbdev, &flags_sched); if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_NOT_PENDING) dev_warn(kbdev->dev, "GPU reset pending at the time of GPU lost event"); atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); kbase_csf_scheduler_spin_unlock(kbdev, flags_sched); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); kbase_synchronize_irqs(kbdev); @@ -1030,60 +912,13 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->protected_mode = false; kbase_pm_update_state(kbdev); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* Cancel any pending HWC dumps */ kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#else - /* Releasing the VM state lock here is safe because - * we are guaranteed to be in either STOPPING_IDLE, - * STOPPING_ACTIVE or SUSPEND_PENDING at this point. - * The only transitions that are valid from here are to - * STOPPED, STOPPED_GPU_REQUESTED or SUSPENDED which can - * only happen at the completion of the GPU lost handling. - */ - mutex_unlock(&arb_vm_state->vm_state_lock); - mutex_lock(&kbdev->pm.lock); - mutex_lock(&arb_vm_state->vm_state_lock); - if (kbase_io_is_gpu_powered(kbdev) && !kbase_io_is_aw_removed(kbdev)) { - ktime_t end_timestamp = ktime_get_raw(); - - /* GPU is no longer mapped to VM. So no interrupts will - * be received and Mali registers have been replaced by - * dummy RAM - */ - WARN(kbase_io_has_gpu(kbdev), "GPU is still available after GPU lost event\n"); - - /* Full GPU reset will have been done by hypervisor, so cancel */ - atomic_set(&kbdev->hwaccess.backend.reset_gpu, KBASE_RESET_GPU_NOT_PENDING); - hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); - - kbase_synchronize_irqs(kbdev); - - /* Clear all jobs running on the GPU */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbdev->protected_mode = false; - kbase_backend_reset(kbdev, &end_timestamp); - kbase_pm_metrics_update(kbdev, NULL); - kbase_pm_update_state(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* Cancel any pending HWC dumps */ - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING || - kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_FAULT) { - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - } - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - } - mutex_unlock(&kbdev->pm.lock); -#endif /* MALI_USE_CSF */ } -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) int kbase_pm_force_mcu_wakeup_after_sleep(struct kbase_device *kbdev) { unsigned long flags; @@ -1108,17 +943,15 @@ static int pm_handle_mcu_sleep_on_runtime_suspend(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->csf.scheduler.lock); lockdep_assert_held(&kbdev->pm.lock); -#ifdef CONFIG_MALI_VALHALL_DEBUG /* In case of no active CSG on slot, powering up L2 could be skipped and * proceed directly to suspend GPU. * ToDo: firmware has to be reloaded after wake-up as no halt command * has been sent when GPU was put to sleep mode. */ - if (!kbase_csf_scheduler_get_nr_active_csgs(kbdev)) + if (IS_ENABLED(CONFIG_MALI_VALHALL_DEBUG) && !kbase_csf_scheduler_get_nr_active_csgs(kbdev)) dev_info( kbdev->dev, "No active CSGs. Can skip the power up of L2 and go for suspension directly"); -#endif ret = kbase_pm_force_mcu_wakeup_after_sleep(kbdev); if (ret) { @@ -1300,5 +1133,3 @@ out: return ret; } - -#endif diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.c index 32c51152cbbd..27ab4162c325 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2013-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -27,19 +27,37 @@ #include #include #include -#include -int kbase_pm_ca_init(struct kbase_device *kbdev) -{ #ifdef CONFIG_MALI_VALHALL_DEVFREQ +static void pm_init_cores_enabled_mask(struct kbase_device *kbdev) +{ struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; if (kbdev->current_core_mask) pm_backend->ca_cores_enabled = kbdev->current_core_mask; else pm_backend->ca_cores_enabled = kbdev->gpu_props.shader_present; +} + +static void pm_init_gov_cores_enabled_mask(struct kbase_device *kbdev) +{ + struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; + + if (kbdev->current_core_mask) + pm_backend->ca_gov_cores_enabled = kbdev->current_core_mask; + else + pm_backend->ca_gov_cores_enabled = kbdev->gpu_props.shader_present; +} #endif +int kbase_pm_ca_init(struct kbase_device *kbdev) +{ +#ifdef CONFIG_MALI_VALHALL_DEVFREQ + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + pm_init_gov_cores_enabled_mask(kbdev); + + pm_init_cores_enabled_mask(kbdev); +#endif return 0; } @@ -48,61 +66,119 @@ void kbase_pm_ca_term(struct kbase_device *kbdev) CSTD_UNUSED(kbdev); } -#ifdef CONFIG_MALI_VALHALL_DEVFREQ -void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) +void kbase_pm_ca_set_gov_core_mask_nolock(struct kbase_device *kbdev, enum mask_type core_mask_type, + u64 core_mask) { struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; - unsigned long flags; -#if MALI_USE_CSF - u64 old_core_mask = 0; - bool mmu_sync_needed = false; - if (!IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) && - kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901)) { - mmu_sync_needed = true; - down_write(&kbdev->csf.mmu_sync_sem); + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + dev_warn( + kbdev->dev, + "This function requires Kbase to have access to GOV_CORE_MASK register, cannot proceed\n"); + return; + } + + lockdep_assert_held(&kbdev->hwaccess_lock); + + /** A value of ZERO means disabling. + * When disabling, store the last used mask for re-enabling + */ + if (core_mask_type == SYSFS_COREMASK) { + if (core_mask != 0) + pm_backend->ca_gov_cores_enabled = core_mask; + else +#ifdef CONFIG_MALI_VALHALL_DEVFREQ + pm_backend->ca_gov_cores_enabled = kbdev->current_core_mask; +#else + pm_backend->ca_gov_cores_enabled = + kbdev->gpu_props.curr_config.shader_present; +#endif + } +#ifdef CONFIG_MALI_VALHALL_DEVFREQ + /* sysfs core mask takes priority over OPP mask when sysfs core mask is set */ + else if (core_mask_type == DEVFREQ_COREMASK) { + if (core_mask == 0) { + dev_warn(kbdev->dev, + "Required core_mask cannot be zero when sysfs usage disabled\n"); + return; + } + /* if sysfs non-zero then no need to re-write value */ + if (kbdev->pm.sysfs_gov_core_mask) + return; + + pm_backend->ca_gov_cores_enabled = core_mask; } #endif - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + /** after all checks, write the to GOV_CORE_MASK register if GPU powered, + * otherwise value will be applied on next reboot. + */ + if (kbase_io_is_gpu_powered(kbdev)) + kbase_reg_write64(kbdev, GPU_GOVERNOR_ENUM(GOV_CORE_MASK), + pm_backend->ca_gov_cores_enabled); +} -#if MALI_USE_CSF - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { - if (kbase_io_is_gpu_powered(kbdev)) { - kbase_reg_write64(kbdev, GPU_GOVERNOR_ENUM(GOV_CORE_MASK), - core_mask & kbdev->pm.debug_core_mask); - } +void kbase_pm_ca_set_gov_core_mask(struct kbase_device *kbdev, enum mask_type core_mask_type, + u64 core_mask) +{ + unsigned long flags; - goto unlock; + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + dev_warn( + kbdev->dev, + "This function requires Kbase to have access to GOV_CORE_MASK register, cannot proceed\n"); + return; } + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_pm_ca_set_gov_core_mask_nolock(kbdev, core_mask_type, core_mask); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +} + +#ifdef CONFIG_MALI_VALHALL_DEVFREQ + +static int set_core_mask_gov(struct kbase_device *kbdev, u64 core_mask) +{ + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + dev_warn( + kbdev->dev, + "This function requires Kbase to have access to GOV_CORE_MASK register, cannot proceed\n"); + return -EIO; + } + + /* Requires a validity check to ensure we don't try to set cores we do not have */ + if ((core_mask & kbdev->gpu_props.shader_present) != core_mask) { + dev_err(kbdev->dev, + "core_mask (%llu) must be a subset of the shader present (%llu)", core_mask, + kbdev->gpu_props.shader_present); + return -EINVAL; + } + + kbase_pm_ca_set_gov_core_mask(kbdev, DEVFREQ_COREMASK, core_mask); + + return 0; +} + +static int set_core_mask_legacy(struct kbase_device *kbdev, u64 core_mask) +{ + struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; + u64 old_core_mask = 0; + unsigned long flags; + + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); if (!(core_mask & kbdev->pm.debug_core_mask)) { dev_err(kbdev->dev, - "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n", + "OPP core mask 0x%llX does not intersect with sysfs debug mask 0x%llX\n", core_mask, kbdev->pm.debug_core_mask); - goto unlock; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return -EINVAL; } old_core_mask = pm_backend->ca_cores_enabled; -#else - if (!(core_mask & kbdev->pm.debug_core_mask_all)) { - dev_err(kbdev->dev, - "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n", - core_mask, kbdev->pm.debug_core_mask_all); - goto unlock; - } - - if (kbase_dummy_job_wa_enabled(kbdev)) { - dev_err_once(kbdev->dev, - "Dynamic core scaling not supported as dummy job WA is enabled"); - goto unlock; - } -#endif /* MALI_USE_CSF */ pm_backend->ca_cores_enabled = core_mask; kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF /* Check if old_core_mask contained the undesired cores and wait * for those cores to get powered down */ @@ -114,33 +190,51 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) } } + return 0; +} + +void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) +{ + bool mmu_sync_needed = false; + int err; + + if (!IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) && + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901)) { + mmu_sync_needed = true; + down_write(&kbdev->csf.mmu_sync_sem); + } + + err = kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT) ? + set_core_mask_gov(kbdev, core_mask) : + set_core_mask_legacy(kbdev, core_mask); if (mmu_sync_needed) up_write(&kbdev->csf.mmu_sync_sem); -#endif - dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n", pm_backend->ca_cores_enabled); - - return; -unlock: - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF - if (mmu_sync_needed) - up_write(&kbdev->csf.mmu_sync_sem); -#endif + if (!err) + dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n", core_mask); } KBASE_EXPORT_TEST_API(kbase_devfreq_set_core_mask); #endif u64 kbase_pm_ca_get_debug_core_mask(struct kbase_device *kbdev) { -#if MALI_USE_CSF return kbdev->pm.debug_core_mask; -#else - return kbdev->pm.debug_core_mask_all; -#endif } KBASE_EXPORT_TEST_API(kbase_pm_ca_get_debug_core_mask); +u64 kbase_pm_ca_get_sysfs_gov_core_mask(struct kbase_device *kbdev) +{ + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + dev_warn( + kbdev->dev, + "This function requires Kbase to have access to GOV_CORE_MASK register, cannot proceed\n"); + return 0; + } + + return kbdev->pm.sysfs_gov_core_mask; +} +KBASE_EXPORT_TEST_API(kbase_pm_ca_get_sysfs_gov_core_mask); + u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) { u64 debug_core_mask = kbase_pm_ca_get_debug_core_mask(kbdev); @@ -160,18 +254,23 @@ u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) return kbdev->gpu_props.curr_config.shader_present & debug_core_mask; #endif } - KBASE_EXPORT_TEST_API(kbase_pm_ca_get_core_mask); +u64 kbase_pm_ca_get_gov_core_mask(struct kbase_device *kbdev) +{ + lockdep_assert_held(&kbdev->hwaccess_lock); + + return kbdev->pm.backend.ca_gov_cores_enabled; +} +KBASE_EXPORT_TEST_API(kbase_pm_ca_get_gov_core_mask); + u64 kbase_pm_ca_get_instr_core_mask(struct kbase_device *kbdev) { lockdep_assert_held(&kbdev->hwaccess_lock); #if IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) return (((1ull) << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1); -#elif MALI_USE_CSF - return kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); #else - return kbdev->pm.backend.pm_shaders_core_mask; + return kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); #endif } diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.h index 37d1020e28ff..8c55252a1a37 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_ca.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -68,6 +68,26 @@ u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev); */ u64 kbase_pm_ca_get_debug_core_mask(struct kbase_device *kbdev); +/** + * kbase_pm_ca_get_sysfs_gov_core_mask - Get the value sysfs has requested to set shader core mask + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This is used only when Kbase has access to the GOV_CORE_MASK register. + * Return: The bit mask of user-selected cores + */ +u64 kbase_pm_ca_get_sysfs_gov_core_mask(struct kbase_device *kbdev); + +/** + * kbase_pm_ca_get_gov_core_mask - Get currently available gov core mask. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * This is used only when Kbase has access to the GOV_CORE_MASK register. + * Return: a mask of the currently available shader cores. + */ +u64 kbase_pm_ca_get_gov_core_mask(struct kbase_device *kbdev); + /** * kbase_pm_ca_update_core_status - Update core status * diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_coarse_demand.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_coarse_demand.c index cd2d65b1e4bc..d9e2432b9ef7 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_coarse_demand.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_coarse_demand.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -59,9 +59,7 @@ const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { coarse_demand_get_core_active, /* get_core_active */ NULL, /* handle_event */ KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ -#if MALI_USE_CSF COARSE_ON_DEMAND_PM_SCHED_FLAGS, /* pm_sched_flags */ -#endif }; KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_defs.h index 67ce9d48656b..0c9ec908d4b4 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_defs.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_defs.h @@ -31,10 +31,6 @@ #include -#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM) -#define KBASE_PM_RUNTIME 1 -#endif - /* Forward definition - see mali_kbase.h */ struct kbase_device; struct kbase_jd_atom; @@ -61,7 +57,6 @@ struct kbase_jd_atom; * * They specify which type of core should be acted on. */ -#if MALI_USE_CSF enum kbase_pm_core_type { KBASE_PM_CORE_L2 = HOST_POWER_ENUM(L2_PRESENT), KBASE_PM_CORE_SHADER = HOST_POWER_ENUM(SHADER_PRESENT), @@ -76,14 +71,6 @@ enum kbase_pm_core_type { */ KBASE_PM_CORE_BASE = HOST_POWER_ENUM(BASE_PRESENT) }; -#else -enum kbase_pm_core_type { - KBASE_PM_CORE_L2 = GPU_CONTROL_ENUM(L2_PRESENT), - KBASE_PM_CORE_SHADER = GPU_CONTROL_ENUM(SHADER_PRESENT), - KBASE_PM_CORE_TILER = GPU_CONTROL_ENUM(TILER_PRESENT), - KBASE_PM_CORE_STACK = GPU_CONTROL_ENUM(STACK_PRESENT) -}; -#endif /* * enum kbase_l2_core_state - The states used for the L2 cache & tiler power @@ -95,7 +82,6 @@ enum kbase_l2_core_state { #undef KBASEP_L2_STATE }; -#if MALI_USE_CSF /* * enum kbase_mcu_state - The states used for the MCU state machine. */ @@ -104,7 +90,6 @@ enum kbase_mcu_state { #include "mali_kbase_pm_mcu_states.h" #undef KBASEP_MCU_STATE }; -#endif /* * enum kbase_shader_core_state - The states used for the shaders' state machine. @@ -163,22 +148,11 @@ enum kbase_pm_runtime_suspend_abort_reason { * time_period_start timestamp, measured in units of 256ns. * @time_in_protm: The amount of time the GPU has spent in protected mode since * the time_period_start timestamp, measured in units of 256ns. - * @busy_cl: the amount of time the GPU was busy executing CL jobs. Note that - * if two CL jobs were active for 256ns, this value would be updated - * with 2 (2x256ns). - * @busy_gl: the amount of time the GPU was busy executing GL jobs. Note that - * if two GL jobs were active for 256ns, this value would be updated - * with 2 (2x256ns). */ struct kbasep_pm_metrics { u32 time_busy; u32 time_idle; -#if MALI_USE_CSF u32 time_in_protm; -#else - u32 busy_cl[2]; - u32 busy_gl; -#endif }; /** @@ -189,11 +163,6 @@ struct kbasep_pm_metrics { * kbase_ipa_control client * @skip_gpu_active_sanity_check: Decide whether to skip GPU_ACTIVE sanity * check in DVFS utilisation calculation - * @gpu_active: true when the GPU is executing jobs. false when - * not. Updated when the job scheduler informs us a job in submitted - * or removed from a GPU slot. - * @active_cl_ctx: number of CL jobs active on the GPU. Array is per-device. - * @active_gl_ctx: number of GL jobs active on the GPU. Array is per-slot. * @lock: spinlock protecting the kbasep_pm_metrics_state structure * @platform_data: pointer to data controlled by platform specific code * @kbdev: pointer to kbase device for which metrics are collected @@ -209,14 +178,8 @@ struct kbasep_pm_metrics { */ struct kbasep_pm_metrics_state { ktime_t time_period_start; -#if MALI_USE_CSF void *ipa_control_client; bool skip_gpu_active_sanity_check; -#else - bool gpu_active; - u32 active_cl_ctx[2]; - u32 active_gl_ctx[3]; -#endif spinlock_t lock; void *platform_data; @@ -336,6 +299,8 @@ union kbase_pm_policy_data { * called previously. * See &struct kbase_pm_callback_conf. * @ca_cores_enabled: Cores that are currently available + * @ca_gov_cores_enabled: Final value used for setting GOV_CORE_MASK register. + * Depends on sysfs-core-mask and devfreq-core-mask. * @apply_hw_issue_TITANHW_2938_wa: Indicates if the workaround for KBASE_HW_ISSUE_TITANHW_2938 * needs to be applied when unmapping memory from GPU. * @mcu_state: The current state of the micro-control unit, only applicable @@ -421,20 +386,6 @@ union kbase_pm_policy_data { * re-enabled whilst the flag is set. * @in_reset: True if a GPU is resetting and normal power manager operation is * suspended - * @partial_shaderoff: True if we want to partial power off shader cores, - * it indicates a partial shader core off case, - * do some special operation for such case like flush - * L2 cache because of GPU2017-861 - * @protected_entry_transition_override : True if GPU reset is being used - * before entering the protected mode and so - * the reset handling behaviour is being - * overridden. - * @protected_transition_override : True if a protected mode transition is in - * progress and is overriding power manager - * behaviour. - * @protected_l2_override : Non-zero if the L2 cache is required during a - * protected mode transition. Has no effect if not - * transitioning. * @hwcnt_desired: True if we want GPU hardware counters to be enabled. * @hwcnt_disabled: True if GPU hardware counters are not enabled. * @hwcnt_disable_work: Work item to disable GPU hardware counters, used if @@ -456,6 +407,7 @@ union kbase_pm_policy_data { * @gpu_clock_control_work: work item to set GPU clock during L2 power cycle * using gpu_clock_control * @reset_in_progress: Set if reset is ongoing, otherwise set to 0 + * @gpu_lost_pending: Set if GPU_LOST is detected and its handling is pending. * * This structure contains data for the power management framework. There is one * instance of this structure per device in the system. @@ -508,16 +460,14 @@ struct kbase_pm_backend_data { void (*callback_power_runtime_gpu_active)(struct kbase_device *kbdev); u64 ca_cores_enabled; + u64 ca_gov_cores_enabled; -#if MALI_USE_CSF bool apply_hw_issue_TITANHW_2938_wa; enum kbase_mcu_state mcu_state; -#endif enum kbase_l2_core_state l2_state; enum kbase_shader_core_state shaders_state; u64 shaders_avail; u64 shaders_desired_mask; -#if MALI_USE_CSF bool mcu_desired; bool policy_change_clamp_state_to_off; bool waiting_for_mmu_fault_handling; @@ -525,8 +475,6 @@ struct kbase_pm_backend_data { struct mutex policy_change_lock; struct workqueue_struct *core_idle_wq; struct work_struct core_idle_work; - -#ifdef KBASE_PM_RUNTIME unsigned long gpu_sleep_allowed; bool gpu_sleep_mode_active; bool exit_gpu_sleep_mode; @@ -534,7 +482,6 @@ struct kbase_pm_backend_data { bool gpu_wakeup_override; bool db_mirror_interrupt_enabled; enum kbase_pm_runtime_suspend_abort_reason runtime_suspend_abort_reason; -#endif /** * @has_host_pwr_iface: GPU supports the host power control interface. @@ -547,21 +494,12 @@ struct kbase_pm_backend_data { */ bool pwr_cntl_delegated; bool l2_force_off_after_mcu_halt; -#endif bool l2_desired; bool l2_always_on; bool shaders_desired; bool in_reset; -#if !MALI_USE_CSF - bool partial_shaderoff; - - bool protected_entry_transition_override; - bool protected_transition_override; - int protected_l2_override; -#endif - bool hwcnt_desired; bool hwcnt_disabled; struct work_struct hwcnt_disable_work; @@ -573,9 +511,10 @@ struct kbase_pm_backend_data { struct work_struct gpu_clock_control_work; atomic_t reset_in_progress; + + bool gpu_lost_pending; }; -#if MALI_USE_CSF /* CSF PM flag, signaling that the MCU shader Core should be kept on */ #define CSF_DYNAMIC_PM_CORE_KEEP_ON (1 << 0) /* CSF PM flag, signaling no scheduler suspension on idle groups */ @@ -591,7 +530,6 @@ struct kbase_pm_backend_data { #if !MALI_CUSTOMER_RELEASE #define ALWAYS_ON_DEMAND_PM_SCHED_FLAGS (CSF_DYNAMIC_PM_SCHED_IGNORE_IDLE) #endif -#endif /* List of policy IDs */ enum kbase_pm_policy_id { @@ -714,14 +652,12 @@ struct kbase_pm_policy { enum kbase_pm_policy_id id; -#if MALI_USE_CSF /* Policy associated with CSF PM scheduling operational flags. * There are pre-defined required flags exist for each of the * ARM released policies, such as 'always_on', 'coarse_demand' * and etc. */ unsigned int pm_sched_flags; -#endif }; #endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_driver.c index 9547c4ca6b17..018d2699c1ac 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_driver.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_driver.c @@ -32,11 +32,7 @@ #include #include -#if MALI_USE_CSF #include -#else -#include -#endif /* !MALI_USE_CSF */ #include #include @@ -47,12 +43,9 @@ #include #include #include -#include #include -#if MALI_USE_CSF #include -#endif #include @@ -110,7 +103,6 @@ static u64 kbase_pm_get_state(struct kbase_device *kbdev, enum kbase_pm_core_typ static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev); -#if MALI_USE_CSF static bool pwr_status_gpu_reset_allowed(struct kbase_device *kbdev) { @@ -173,107 +165,26 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev) if (kbase_pm_no_mcu_core_pwroff(kbdev) && kbdev->pm.backend.l2_desired) return true; -#ifdef KBASE_PM_RUNTIME - if (kbdev->pm.backend.gpu_wakeup_override || - kbdev->pm.backend.runtime_suspend_abort_reason != ABORT_REASON_NONE) - return true; -#endif + if (IS_ENABLED(CONFIG_PM)) { + return kbdev->pm.backend.gpu_wakeup_override || + kbdev->pm.backend.runtime_suspend_abort_reason != ABORT_REASON_NONE; + } return false; } -#endif /* MALI_USE_CSF */ bool kbase_pm_is_l2_desired(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - if (kbdev->pm.backend.protected_entry_transition_override) - return false; - - if (kbdev->pm.backend.protected_transition_override && - kbdev->pm.backend.protected_l2_override) - return true; - - if (kbdev->pm.backend.protected_transition_override && !kbdev->pm.backend.shaders_desired) - return false; -#else if (unlikely(kbdev->pm.backend.policy_change_clamp_state_to_off)) return false; /* Power up the L2 cache only when MCU is desired */ if (likely(kbdev->csf.firmware_inited)) return kbase_pm_is_mcu_desired(kbdev); -#endif return kbdev->pm.backend.l2_desired; } -#if !MALI_USE_CSF -void kbase_pm_protected_override_enable(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - kbdev->pm.backend.protected_transition_override = true; -} -void kbase_pm_protected_override_disable(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - kbdev->pm.backend.protected_transition_override = false; -} - -int kbase_pm_protected_entry_override_enable(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - WARN_ON(!kbdev->protected_mode_transition); - - if (kbdev->pm.backend.l2_always_on && (kbdev->system_coherency == COHERENCY_ACE)) { - WARN_ON(kbdev->pm.backend.protected_entry_transition_override); - - /* - * If there is already a GPU reset pending then wait for it to - * complete before initiating a special reset for protected - * mode entry. - */ - if (kbase_reset_gpu_silent(kbdev)) - return -EAGAIN; - - kbdev->pm.backend.protected_entry_transition_override = true; - } - - return 0; -} - -void kbase_pm_protected_entry_override_disable(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - WARN_ON(!kbdev->protected_mode_transition); - - if (kbdev->pm.backend.l2_always_on && (kbdev->system_coherency == COHERENCY_ACE)) { - WARN_ON(!kbdev->pm.backend.protected_entry_transition_override); - - kbdev->pm.backend.protected_entry_transition_override = false; - } -} - -void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (override) { - kbdev->pm.backend.protected_l2_override++; - WARN_ON(kbdev->pm.backend.protected_l2_override <= 0); - } else { - kbdev->pm.backend.protected_l2_override--; - WARN_ON(kbdev->pm.backend.protected_l2_override < 0); - } - - kbase_pm_update_state(kbdev); -} -#endif - -#if MALI_USE_CSF #define ACTION_TYPE_TO_REG_HOST_POWER(core_name) \ { \ switch (action) { \ @@ -299,7 +210,6 @@ void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override) break; \ } \ } -#endif #define ACTION_TYPE_TO_REG_GPU_CONTROL(core_name) \ { \ @@ -348,7 +258,6 @@ static u32 core_type_to_reg(struct kbase_device *kbdev, enum kbase_pm_core_type { u32 reg = 0x0; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { switch (core_type) { case KBASE_PM_CORE_L2: @@ -392,9 +301,7 @@ static u32 core_type_to_reg(struct kbase_device *kbdev, enum kbase_pm_core_type dev_err(kbdev->dev, "Invalid core type for host power control"); break; } - } else -#endif - { + } else { switch (core_type) { case KBASE_PM_CORE_L2: ACTION_TYPE_TO_REG_GPU_CONTROL(L2); @@ -438,75 +345,6 @@ static u32 core_type_to_reg(struct kbase_device *kbdev, enum kbase_pm_core_type return reg; } -#if !MALI_USE_CSF -/** - * map_core_type_to_tl_pm_state - Map core type to TL_PM_STATE. - * - * @kbdev: The kbase device for the core - * @core_type: The type of core - * - * Given a core_type (defined by kbase_pm_core_type) this function will return - * TL_PM_STATE_*, which is a mapping of core_type to respective core type timeline value. - * - * Return: Core type timeline value. - */ -__pure static u32 map_core_type_to_tl_pm_state(struct kbase_device *kbdev, - enum kbase_pm_core_type core_type) -{ - u32 map = 0; - - switch (core_type) { - case KBASE_PM_CORE_L2: - map = TL_PM_STATE_L2; - break; - case KBASE_PM_CORE_SHADER: - map = TL_PM_STATE_SHADER; - break; - case KBASE_PM_CORE_TILER: - map = TL_PM_STATE_TILER; - break; - case KBASE_PM_CORE_STACK: - if (corestack_driver_control) - map = TL_PM_STATE_STACK; - break; - default: - dev_err(kbdev->dev, "Invalid core type"); - } - - /* Core stack might not change default value */ - WARN_ON(!map); - return map; -} -#endif - -#if IS_ENABLED(CONFIG_ARM64) && !MALI_USE_CSF - -static void mali_cci_flush_l2(struct kbase_device *kbdev) -{ - u32 val; - const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; - const u32 timeout_us = - kbase_get_timeout_ms(kbdev, KBASE_CLEAN_CACHE_TIMEOUT) * USEC_PER_MSEC; - - /* - * Note that we don't take the cache flush mutex here since - * we expect to be the last user of the L2, all other L2 users - * would have dropped their references, to initiate L2 power - * down, L2 power down being the only valid place for this - * to be called from. - */ - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_CACHE_CLN_INV_L2); - - /* Wait for cache flush to complete before continuing, exit on - * gpu resets or loop expiry. - */ - kbase_reg_poll32_timeout(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT), val, val & mask, 0, - timeout_us, false); -} -#endif - -#if MALI_USE_CSF /** * pwr_cmd_constructor - Construct an on or off power command for a core type. * @@ -563,7 +401,6 @@ static u64 pwr_cmd_constructor(struct kbase_device *kbdev, enum kbase_pm_core_ty return pwr_cmd; } -#endif /* MALI_USE_CSF */ /** * kbase_pm_invoke - Invokes an action on a core set @@ -585,20 +422,6 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, enum kbase_pm_core_type reg = core_type_to_reg(kbdev, core_type, action); -#if !MALI_USE_CSF - if (cores) { - u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY); - - if (action == ACTION_PWRON) - state |= cores; - else if (action == ACTION_PWROFF) - state &= ~cores; - - KBASE_TLSTREAM_AUX_PM_STATE(kbdev, map_core_type_to_tl_pm_state(kbdev, core_type), - state); - } - -#endif /* Tracing */ if (cores) { if (action == ACTION_PWRON) @@ -612,11 +435,9 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, enum kbase_pm_core_type case KBASE_PM_CORE_L2: KBASE_KTRACE_ADD(kbdev, PM_PWRON_L2, NULL, cores); break; -#if MALI_USE_CSF case KBASE_PM_CORE_NEURAL: KBASE_KTRACE_ADD(kbdev, PM_PWRON_NEURAL, NULL, cores); break; -#endif default: break; } @@ -633,23 +454,15 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, enum kbase_pm_core_type /* disable snoops before L2 is turned off */ kbase_pm_cache_snoop_disable(kbdev); break; -#if MALI_USE_CSF case KBASE_PM_CORE_NEURAL: KBASE_KTRACE_ADD(kbdev, PM_PWROFF_NEURAL, NULL, cores); break; -#endif default: break; } } - if (kbase_dummy_job_wa_enabled(kbdev) && action == ACTION_PWRON && - core_type == KBASE_PM_CORE_SHADER && - !(kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER)) { - kbase_dummy_job_wa_execute(kbdev, cores); - } -#if MALI_USE_CSF - else if (kbdev->pm.backend.has_host_pwr_iface) { + if (kbdev->pm.backend.has_host_pwr_iface) { kbase_reg_write64(kbdev, reg, cores); if (reg == HOST_POWER_ENUM(PWR_CMDARG)) { @@ -657,9 +470,7 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, enum kbase_pm_core_type kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_COMMAND), pwr_cmd); } - } -#endif /* MALI_USE_CSF */ - else + } else kbase_reg_write64(kbdev, reg, cores); } @@ -707,12 +518,10 @@ u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, enum kbase_pm_core_ty return kbdev->gpu_props.tiler_present; case KBASE_PM_CORE_STACK: return kbdev->gpu_props.stack_present; -#if MALI_USE_CSF case KBASE_PM_CORE_NEURAL: return kbdev->gpu_props.neural_present; case KBASE_PM_CORE_BASE: return kbdev->gpu_props.base_present; -#endif default: break; } @@ -782,11 +591,9 @@ u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev, enum kbase_pm_core_type case KBASE_PM_CORE_L2: KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, result); break; -#if MALI_USE_CSF case KBASE_PM_CORE_NEURAL: KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_NEURAL, NULL, result); break; -#endif default: break; } @@ -823,13 +630,11 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_L2_CONFIG)) return; -#if MALI_USE_CSF if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) { val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG)); kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG), L2_CONFIG_PBHA_HWU_SET(val, kbdev->pbha_propagate_bits)); } -#endif /* MALI_USE_CSF */ /* * Skip if size and hash are not given explicitly, @@ -851,7 +656,6 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) val &= ~L2_CONFIG_HASH_MASK; val |= (kbdev->l2_hash_override << L2_CONFIG_HASH_SHIFT); } else if (kbdev->l2_hash_values_override) { -#if MALI_USE_CSF uint i; WARN_ON(!kbase_hw_has_l2_slice_hash_feature(kbdev)); @@ -869,7 +673,6 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_L2_SLICE_HASH_OFFSET(i), kbdev->l2_hash_values[i]); } -#endif /* MALI_USE_CSF */ } dev_dbg(kbdev->dev, "Program 0x%x to L2_CONFIG\n", val); @@ -885,7 +688,6 @@ static void kbase_pm_control_gpu_clock(struct kbase_device *kbdev) queue_work(system_wq, &backend->gpu_clock_control_work); } -#if MALI_USE_CSF static const char *kbase_mcu_state_to_string(enum kbase_mcu_state state) { const char *const strings[] = { @@ -951,8 +753,6 @@ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state s return ((state == KBASE_MCU_OFF) || (state == KBASE_MCU_IN_SLEEP)); } -#ifdef KBASE_PM_RUNTIME - void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev) { u32 val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL)); @@ -1052,7 +852,6 @@ static void wait_mcu_as_inactive(struct kbase_device *kbdev) if (kbase_prepare_to_reset_gpu(kbdev, 0)) kbase_reset_gpu(kbdev); } -#endif /** * kbasep_pm_toggle_power_interrupt - Toggles the IRQ mask for power interrupts @@ -1072,14 +871,12 @@ static void kbasep_pm_toggle_power_interrupt(struct kbase_device *kbdev, bool en lockdep_assert_held(&kbdev->hwaccess_lock); -#if MALI_USE_CSF /* No toggling is needed when Host control power interface is there, as PM actions * done by the firmware for Tiler, shader, neural won't generate the POWER_CHANGED * irq on Host side. */ if (kbdev->pm.backend.has_host_pwr_iface) return; -#endif irq_mask = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)); @@ -1093,7 +890,6 @@ static void kbasep_pm_toggle_power_interrupt(struct kbase_device *kbdev, bool en kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), irq_mask); } -#if MALI_USE_CSF /** * hctl_neural_engines_active - Check the neural engines are active * @@ -1212,9 +1008,7 @@ static bool hctl_base_power_down_done(struct kbase_device *kbdev, u64 shaders_re return (!base_trans && base_ready == shaders_ready); } -#endif /* MALI_USE_CSF */ -#if MALI_USE_CSF /** * hctl_shader_cores_active - Check the shader cores are active * @@ -1334,7 +1128,6 @@ static bool hctl_cores_power_down_done(struct kbase_device *kbdev, u64 cores_rea { return (!cores_trans && (cores_ready == cores_avail)); } -#endif /* MALI_USE_CSF */ static void disable_gpu_idle_timer_no_db(struct kbase_device *kbdev) { @@ -1392,16 +1185,20 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) bases_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_BASE); } /* This check mirrors the check inside kbase_pm_l2_update_state(). */ + if (unlikely(!kbase_io_has_gpu(kbdev))) { - backend->hwcnt_desired = false; - if (!backend->hwcnt_disabled) { - backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; - KBASE_KTRACE_ADD(kbdev, PM_MCU_ON_HWCNT_DISABLE, NULL, - backend->mcu_state); - kbase_pm_trigger_hwcnt_disable(kbdev); + if (kbdev->csf.firmware_hctl_core_pwr) { + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) { + backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; + KBASE_KTRACE_ADD(kbdev, PM_MCU_ON_HWCNT_DISABLE, NULL, + backend->mcu_state); + kbase_pm_trigger_hwcnt_disable(kbdev); + } } - if (backend->hwcnt_disabled && (backend->mcu_state != KBASE_MCU_OFF)) { + if (backend->mcu_state != KBASE_MCU_OFF && + (backend->hwcnt_disabled || !kbdev->csf.firmware_hctl_core_pwr)) { backend->mcu_state = KBASE_MCU_OFF; KBASE_KTRACE_ADD(kbdev, PM_MCU_OFF, NULL, backend->mcu_state); dev_dbg(kbdev->dev, "GPU lost has occurred - MCU off"); @@ -1520,20 +1317,22 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) break; case KBASE_MCU_ON_HWCNT_ENABLE: - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { - unsigned long flags; + /* This state is only needed for the case of the shader power control + * under Host. + * HWCNT is left enabled throughout MCU power cycles in the opposite case. + */ + if (kbdev->csf.firmware_hctl_core_pwr) { + backend->hwcnt_desired = true; + if (backend->hwcnt_disabled) { + unsigned long flags; - kbase_csf_scheduler_spin_lock(kbdev, &flags); - kbase_hwcnt_backend_csf_set_hw_availability( - &kbdev->hwcnt_gpu_iface, - kbdev->gpu_props.curr_config.l2_slices, - kbdev->gpu_props.curr_config.shader_present, - kbdev->pm.debug_core_mask); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbase_csf_scheduler_spin_unlock(kbdev, flags); - backend->hwcnt_disabled = false; + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + backend->hwcnt_disabled = false; + } } + backend->mcu_state = KBASE_MCU_ON; break; @@ -1547,6 +1346,28 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) backend->hwcnt_desired = false; if (!backend->hwcnt_disabled) kbase_pm_trigger_hwcnt_disable(kbdev); + + /* WA to avoid a double call to + * kbase_hwcnt_backend_csf_on_after_mcu_on. + * If MCU reload hasn't been triggered yet, + * firmware_reloaded will be set to false. This will lead to + * a call to kbase_csf_firmware_reload_completed() on the + * following FW global IRQ. This call should be a no-op. + * However, HWC needs a MCU_OFF notification here to pair up + * with the first MCU_ON call made during initial FW boot. + * + * Note: On NO_MALI, there is no call to + * kbase_hwcnt_backend_csf_on_after_mcu_on during + * firmware init. First call is made on + * kbase_csf_firmware_reload_completed() during the first + * FW global IRQ. Hence, there is no need to pair up the + * MCU power state notifications to HWC. + */ + if (!IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) && + !kbdev->csf.firmware_reloaded) + kbase_hwcnt_backend_csf_on_after_mcu_off( + &kbdev->hwcnt_gpu_iface); + backend->mcu_state = KBASE_MCU_HCTL_MCU_ON_RECHECK; } } else if (kbase_pm_handle_mcu_core_attr_update(kbdev)) @@ -1566,7 +1387,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) case KBASE_MCU_HCTL_MCU_ON_RECHECK: backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); - + WARN_ON(!kbdev->csf.firmware_hctl_core_pwr); if (!backend->hwcnt_disabled) { /* Wait for being disabled */ ; @@ -1640,17 +1461,16 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; break; } + if (kbdev->csf.firmware_hctl_core_pwr) { + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) + kbase_pm_trigger_hwcnt_disable(kbdev); + } - backend->hwcnt_desired = false; - if (!backend->hwcnt_disabled) - kbase_pm_trigger_hwcnt_disable(kbdev); - - if (backend->hwcnt_disabled) { -#ifdef KBASE_PM_RUNTIME - if (backend->gpu_sleep_mode_active) + if (backend->hwcnt_disabled || !kbdev->csf.firmware_hctl_core_pwr) { + if (IS_ENABLED(CONFIG_PM) && backend->gpu_sleep_mode_active) backend->mcu_state = KBASE_MCU_ON_SLEEP_INITIATE; else { -#endif backend->mcu_state = KBASE_MCU_ON_HALT; #if IS_ENABLED(CONFIG_MALI_VALHALL_CORESIGHT) kbase_debug_coresight_csf_state_request( @@ -1706,6 +1526,9 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) if (kbase_csf_firmware_mcu_halt_req_complete(kbdev)) { KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_MCU_HALTED, NULL, kbase_csf_ktrace_gpu_cycle_cnt(kbdev)); + + kbase_hwcnt_backend_csf_on_after_mcu_off(&kbdev->hwcnt_gpu_iface); + if (kbdev->csf.firmware_hctl_core_pwr) backend->mcu_state = KBASE_MCU_HCTL_SHADERS_READY_OFF; else @@ -1766,7 +1589,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) kbasep_pm_toggle_power_interrupt(kbdev, true); backend->mcu_state = KBASE_MCU_OFF; break; -#ifdef KBASE_PM_RUNTIME + case KBASE_MCU_ON_SLEEP_INITIATE: if (!kbase_pm_is_mcu_desired(kbdev)) handle_sleep_initiate_state(kbdev); @@ -1778,6 +1601,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) if (kbase_pm_is_mcu_desired(kbdev)) { /* Assume the transition is complete and prepare to goto ON state */ WARN_ON_ONCE(backend->l2_state != KBASE_L2_ON); + kbase_hwcnt_backend_csf_on_after_mcu_off(&kbdev->hwcnt_gpu_iface); backend->mcu_state = KBASE_MCU_IN_SLEEP; break; } @@ -1787,6 +1611,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) if (kbase_csf_firmware_is_mcu_in_sleep(kbdev)) { KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_MCU_SLEEP, NULL, kbase_csf_ktrace_gpu_cycle_cnt(kbdev)); + kbase_hwcnt_backend_csf_on_after_mcu_off(&kbdev->hwcnt_gpu_iface); backend->mcu_state = KBASE_MCU_IN_SLEEP; kbase_pm_enable_db_mirror_interrupt(kbdev); kbase_csf_scheduler_reval_idleness_post_sleep(kbdev); @@ -1827,15 +1652,20 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) */ if (!kbdev->csf.firmware_hctl_core_pwr) kbasep_pm_toggle_power_interrupt(kbdev, false); + backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); } break; -#endif /* KBASE_PM_RUNTIME */ + case KBASE_MCU_RESET_WAIT: /* Reset complete */ - if (!backend->in_reset) + if (!backend->in_reset) { + kbase_hwcnt_backend_csf_on_after_mcu_off_reset( + &kbdev->hwcnt_gpu_iface); backend->mcu_state = KBASE_MCU_OFF; + } #if IS_ENABLED(CONFIG_MALI_VALHALL_CORESIGHT) kbdev->csf.coresight.disable_on_pmode_enter = false; @@ -1886,7 +1716,6 @@ static void core_idle_worker(struct work_struct *work) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -#endif static const char *kbase_l2_core_state_to_string(enum kbase_l2_core_state state) { @@ -1919,23 +1748,6 @@ static void kbase_ktrace_log_l2_core_state(struct kbase_device *kbdev, #endif } -#if !MALI_USE_CSF -/* On powering on the L2, the tracked kctx becomes stale and can be cleared. - * This enables the backend to spare the START_FLUSH.INV_SHADER_OTHER - * operation on the first submitted katom after the L2 powering on. - */ -static void kbase_pm_l2_clear_backend_slot_submit_kctx(struct kbase_device *kbdev) -{ - int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* Clear the slots' last katom submission kctx */ - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) - kbdev->hwaccess.backend.slot_rb[js].last_kctx_tagged = SLOT_RB_NULL_TAG_VAL; -} -#endif - static bool can_power_down_l2(struct kbase_device *kbdev) { lockdep_assert_held(&kbdev->hwaccess_lock); @@ -1954,18 +1766,12 @@ static bool can_power_up_l2(struct kbase_device *kbdev) static bool need_tiler_control(struct kbase_device *kbdev) { -#if MALI_USE_CSF if (kbase_pm_no_mcu_core_pwroff(kbdev)) return true; else return false; -#else - CSTD_UNUSED(kbdev); - return true; -#endif } -#if MALI_USE_CSF /** * hctl_l2_power_down - Initiate power down of L2 cache * @@ -1994,7 +1800,6 @@ static void hctl_l2_power_down(struct kbase_device *kbdev) kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, kbdev->gpu_props.curr_config.l2_present, ACTION_PWROFF); } -#endif /** * hctl_tiler_power_up_done - Check and/or initiate power up of Tiler @@ -2018,12 +1823,10 @@ static bool hctl_tiler_power_up_done(struct kbase_device *kbdev) return false; if (!tiler_ready) { -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, kbdev->gpu_props.tiler_present, ACTION_PWRON); } -#endif return false; } @@ -2031,8 +1834,11 @@ static bool hctl_tiler_power_up_done(struct kbase_device *kbdev) return true; } -#if MALI_USE_CSF +#if MALI_UNIT_TEST +int delegate_pm_domain_control_to_fw(struct kbase_device *kbdev, u32 pm_domain) +#else static int delegate_pm_domain_control_to_fw(struct kbase_device *kbdev, u32 pm_domain) +#endif { u64 val; int err; @@ -2058,10 +1864,17 @@ static int delegate_pm_domain_control_to_fw(struct kbase_device *kbdev, u32 pm_d if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) kbase_reset_gpu(kbdev); - return -ETIMEDOUT; + return err; } +#if MALI_UNIT_TEST +KBASE_EXPORT_TEST_API(delegate_pm_domain_control_to_fw); +#endif +#if MALI_UNIT_TEST +int retract_pm_domain_control_from_fw(struct kbase_device *kbdev, u32 pm_domain) +#else static int retract_pm_domain_control_from_fw(struct kbase_device *kbdev, u32 pm_domain) +#endif { u64 val; int err; @@ -2085,8 +1898,11 @@ static int retract_pm_domain_control_from_fw(struct kbase_device *kbdev, u32 pm_ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) kbase_reset_gpu(kbdev); - return -ETIMEDOUT; + return err; } +#if MALI_UNIT_TEST +KBASE_EXPORT_TEST_API(retract_pm_domain_control_from_fw); +#endif /** * delegate_pm_domains_control_to_fw - Delegate power control for Tiler, Shading engine and @@ -2206,16 +2022,14 @@ int kbase_pm_get_domain_status(struct kbase_device *kbdev, u32 pm_domain, u32 en dev_err(kbdev->dev, "Inspect command for pm domain %u failed", pm_domain); return err; } -#endif /* MALI_USE_CSF */ +KBASE_EXPORT_TEST_API(kbase_pm_get_domain_status); static int kbase_pm_l2_update_state(struct kbase_device *kbdev) { struct kbase_pm_backend_data *backend = &kbdev->pm.backend; u64 l2_present = kbdev->gpu_props.curr_config.l2_present; u64 tiler_present = kbdev->gpu_props.tiler_present; -#if MALI_USE_CSF const bool has_host_pwr_iface = backend->has_host_pwr_iface; -#endif bool l2_power_up_done; enum kbase_l2_core_state prev_state; @@ -2232,22 +2046,25 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) */ if (!kbase_io_has_gpu(kbdev)) { backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; - backend->hwcnt_desired = false; - if (!backend->hwcnt_disabled) { - /* Don't progress until hw counters are disabled - * This may involve waiting for a worker to complete. - * The HW counters backend disable code checks for the - * GPU removed case and will error out without touching - * the hardware. This step is needed to keep the HW - * counters in a consistent state after a GPU lost. - */ - backend->l2_state = KBASE_L2_ON_HWCNT_DISABLE; - KBASE_KTRACE_ADD(kbdev, PM_L2_ON_HWCNT_DISABLE, NULL, - backend->l2_state); - kbase_pm_trigger_hwcnt_disable(kbdev); + if (kbdev->csf.firmware_hctl_core_pwr) { + backend->hwcnt_desired = false; + if (!backend->hwcnt_disabled) { + /* Don't progress until hw counters are disabled + * This may involve waiting for a worker to complete. + * The HW counters backend disable code checks for the + * GPU removed case and will error out without touching + * the hardware. This step is needed to keep the HW + * counters in a consistent state after a GPU lost. + */ + backend->l2_state = KBASE_L2_ON_HWCNT_DISABLE; + KBASE_KTRACE_ADD(kbdev, PM_L2_ON_HWCNT_DISABLE, NULL, + backend->l2_state); + kbase_pm_trigger_hwcnt_disable(kbdev); + } } - if (backend->hwcnt_disabled && (backend->l2_state != KBASE_L2_OFF)) { + if ((backend->hwcnt_disabled || !kbdev->csf.firmware_hctl_core_pwr) && + (backend->l2_state != KBASE_L2_OFF)) { backend->l2_state = KBASE_L2_OFF; KBASE_KTRACE_ADD(kbdev, PM_L2_OFF, NULL, backend->l2_state); dev_dbg(kbdev->dev, "GPU lost has occurred - L2 off\n"); @@ -2265,14 +2082,14 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) switch (backend->l2_state) { case KBASE_L2_OFF: if (kbase_pm_is_l2_desired(kbdev) && can_power_up_l2(kbdev)) { -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - /* Enable HW timer of IPA control before - * L2 cache is powered-up. - */ - { - kbase_ipa_control_handle_gpu_sleep_exit(kbdev); + if (IS_ENABLED(CONFIG_PM)) { + /* Enable HW timer of IPA control before + * L2 cache is powered-up. + */ + { + kbase_ipa_control_handle_gpu_sleep_exit(kbdev); + } } -#endif /* * Set the desired config for L2 before * powering it on @@ -2287,7 +2104,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) * L2 cache. */ if (need_tiler_control(kbdev)) { -#if MALI_USE_CSF if (backend->pwr_cntl_delegated) { retract_pm_domains_control_from_fw(kbdev); backend->pwr_cntl_delegated = false; @@ -2300,30 +2116,17 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present, ACTION_PWRON); } else -#endif kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, tiler_present, ACTION_PWRON); } else { -#if MALI_USE_CSF if (has_host_pwr_iface && !backend->pwr_cntl_delegated) { delegate_pm_domains_control_to_fw(kbdev); backend->pwr_cntl_delegated = true; } -#endif kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present, ACTION_PWRON); } -#if !MALI_USE_CSF - /* If we have more than one L2 cache then we - * must power them on explicitly. - */ - if (l2_present != 1) - kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present & ~1ULL, - ACTION_PWRON); - /* Clear backend slot submission kctx */ - kbase_pm_l2_clear_backend_slot_submit_kctx(kbdev); -#endif backend->l2_state = KBASE_L2_PEND_ON; } break; @@ -2395,30 +2198,15 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) break; case KBASE_L2_ON_HWCNT_ENABLE: -#if !MALI_USE_CSF - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - backend->hwcnt_disabled = false; - } -#endif backend->l2_state = KBASE_L2_ON; break; case KBASE_L2_ON: if (!kbase_pm_is_l2_desired(kbdev)) { -#if !MALI_USE_CSF - /* Do not power off L2 until the shaders and - * core stacks are off. - */ - if (backend->shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) - break; -#else /* Do not power off L2 until the MCU has been stopped */ if ((backend->mcu_state != KBASE_MCU_OFF) && (backend->mcu_state != KBASE_MCU_IN_SLEEP)) break; -#endif /* We need to make sure hardware counters are * disabled before powering down the L2, to @@ -2435,38 +2223,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) break; case KBASE_L2_ON_HWCNT_DISABLE: -#if !MALI_USE_CSF - /* If the L2 became desired while we were waiting on the - * worker to do the actual hwcnt disable (which might - * happen if some work was submitted immediately after - * the shaders powered off), then we need to early-out - * of this state and re-enable hwcnt. - * - * If we get lucky, the hwcnt disable might not have - * actually started yet, and the logic in the hwcnt - * enable state will prevent the worker from - * performing the disable entirely, preventing loss of - * any hardware counter data. - * - * If the hwcnt disable has started, then we'll lose - * a tiny amount of hardware counter data between the - * disable and the re-enable occurring. - * - * This loss of data is preferable to the alternative, - * which is to block the shader cores from doing any - * work until we're sure hwcnt has been re-enabled. - */ - if (kbase_pm_is_l2_desired(kbdev)) { - backend->l2_state = KBASE_L2_ON_HWCNT_ENABLE; - break; - } - - backend->hwcnt_desired = false; - if (!backend->hwcnt_disabled) - kbase_pm_trigger_hwcnt_disable(kbdev); -#endif - - if (backend->hwcnt_disabled) { + if (backend->hwcnt_disabled || !kbdev->csf.firmware_hctl_core_pwr) { if (kbdev->pm.backend.gpu_clock_slow_down_wa) backend->l2_state = KBASE_L2_SLOW_DOWN_CLOCKS; else @@ -2503,7 +2260,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) backend->l2_state = KBASE_L2_PEND_ON; else if (can_power_down_l2(kbdev)) { if (!backend->l2_always_on) { -#if MALI_USE_CSF if (need_tiler_control(kbdev) && has_host_pwr_iface) { /* If using host power control, * the tiler must be explicitly turned off @@ -2512,7 +2268,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) kbase_pm_invoke(kbdev, KBASE_PM_CORE_TILER, tiler_present, ACTION_PWROFF); } else -#endif /* Powering off the L2 will also power off the tiler. */ kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present, ACTION_PWROFF); @@ -2524,36 +2279,31 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) */ kbase_gpu_start_cache_clean_nolock( kbdev, GPU_COMMAND_CACHE_CLN_INV_L2); -#if !MALI_USE_CSF - KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u); -#else KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL, 0u); -#endif backend->l2_state = KBASE_L2_PEND_OFF; } break; case KBASE_L2_PEND_OFF: if (likely(!backend->l2_always_on)) { -#if MALI_USE_CSF if (need_tiler_control(kbdev) && l2_ready) { hctl_l2_power_down(kbdev); break; } -#endif if (l2_trans || l2_ready) break; } else if (kbdev->cache_clean_in_progress) break; -#if MALI_USE_CSF -#if defined(KBASE_PM_RUNTIME) + + if (IS_ENABLED(CONFIG_PM)) { /* Allow clock gating within the GPU and prevent it - * from being seen as active during sleep. - */ - { - kbase_ipa_control_handle_gpu_sleep_enter(kbdev); + * from being seen as active during sleep. + */ + { + kbase_ipa_control_handle_gpu_sleep_enter(kbdev); + } } -#endif + /* Disabling MCU after L2 cache power down is to address * KBASE_HW_ISSUE_TITANHW_2922 hardware issue. */ @@ -2562,7 +2312,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) WARN_ON_ONCE(backend->mcu_state != KBASE_MCU_OFF); backend->l2_force_off_after_mcu_halt = false; } -#endif /* L2 is now powered off */ backend->l2_state = KBASE_L2_OFF; @@ -2571,9 +2320,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) case KBASE_L2_RESET_WAIT: /* Reset complete */ if (!backend->in_reset) { -#if MALI_USE_CSF backend->l2_force_off_after_mcu_halt = false; -#endif backend->l2_state = KBASE_L2_OFF; } @@ -2597,12 +2344,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) if (kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off && backend->l2_state == KBASE_L2_OFF) { kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off = false; -#if MALI_USE_CSF kbase_csf_scheduler_enqueue_power_off_work(kbdev); -#else /* !MALI_USE_CSF */ - queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq, - &kbdev->pm.backend.gpu_poweroff_wait_work); -#endif /* MALI_USE_CSF */ } return 0; @@ -2666,350 +2408,6 @@ static void shader_poweroff_timer_queue_cancel(struct kbase_device *kbdev) } } -#if !MALI_USE_CSF -static const char *kbase_shader_core_state_to_string(enum kbase_shader_core_state state) -{ - const char *const strings[] = { -#define KBASEP_SHADER_STATE(n) #n, -#include "mali_kbase_pm_shader_states.h" -#undef KBASEP_SHADER_STATE - }; - if (WARN_ON((size_t)state >= ARRAY_SIZE(strings))) - return "Bad shader core state"; - else - return strings[state]; -} - -static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) -{ - struct kbase_pm_backend_data *backend = &kbdev->pm.backend; - struct kbasep_pm_tick_timer_state *stt = &kbdev->pm.backend.shader_tick_timer; - enum kbase_shader_core_state prev_state; - u64 stacks_avail = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (corestack_driver_control) - /* Always power on all the corestacks. Disabling certain - * corestacks when their respective shaders are not in the - * available bitmap is not currently supported. - */ - stacks_avail = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_STACK); - - do { - u64 shaders_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER); - u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); - u64 stacks_trans = 0; - u64 stacks_ready = 0; - - if (corestack_driver_control) { - stacks_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_STACK); - stacks_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK); - } - - /* - * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores - * are vulnerable to corruption if gpu is lost - */ - if (!kbase_io_has_gpu(kbdev)) { - backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; - dev_dbg(kbdev->dev, "GPU lost has occurred - shaders off\n"); - break; - } - - /* mask off ready from trans in case transitions finished - * between the register reads - */ - shaders_trans &= ~shaders_ready; - stacks_trans &= ~stacks_ready; - - prev_state = backend->shaders_state; - - switch (backend->shaders_state) { - case KBASE_SHADERS_OFF_CORESTACK_OFF: - /* Ignore changes to the shader core availability - * except at certain points where we can handle it, - * i.e. off and SHADERS_ON_CORESTACK_ON. - */ - backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); - backend->pm_shaders_core_mask = 0; - - if (backend->shaders_desired && backend->l2_state == KBASE_L2_ON) { - if (backend->hwcnt_desired && !backend->hwcnt_disabled) { - /* Trigger a hwcounter dump */ - backend->hwcnt_desired = false; - kbase_pm_trigger_hwcnt_disable(kbdev); - } - - if (backend->hwcnt_disabled) { - if (corestack_driver_control) { - kbase_pm_invoke(kbdev, KBASE_PM_CORE_STACK, - stacks_avail, ACTION_PWRON); - } - backend->shaders_state = - KBASE_SHADERS_OFF_CORESTACK_PEND_ON; - } - } - break; - - case KBASE_SHADERS_OFF_CORESTACK_PEND_ON: - if (!stacks_trans && stacks_ready == stacks_avail) { - backend->shaders_avail = backend->shaders_desired_mask; - kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, backend->shaders_avail, - ACTION_PWRON); - - if (backend->pm_current_policy && - backend->pm_current_policy->handle_event) - backend->pm_current_policy->handle_event( - kbdev, KBASE_PM_POLICY_EVENT_POWER_ON); - - backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_PEND_ON_CORESTACK_ON: - if (!shaders_trans && shaders_ready == backend->shaders_avail) { - KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, - shaders_ready); - backend->pm_shaders_core_mask = shaders_ready; - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { -#if MALI_USE_CSF - unsigned long flags; - - kbase_csf_scheduler_spin_lock(kbdev, &flags); -#endif - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -#if MALI_USE_CSF - kbase_csf_scheduler_spin_unlock(kbdev, flags); -#endif - backend->hwcnt_disabled = false; - } - - backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_ON_CORESTACK_ON: - backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); - - /* If shaders to change state, trigger a counter dump */ - if (!backend->shaders_desired || - (backend->shaders_desired_mask != shaders_ready)) { - backend->hwcnt_desired = false; - if (!backend->hwcnt_disabled) - kbase_pm_trigger_hwcnt_disable(kbdev); - backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK; - } - break; - - case KBASE_SHADERS_ON_CORESTACK_ON_RECHECK: - backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); - - if (!backend->hwcnt_disabled) { - /* Wait for being disabled */ - ; - } else if (!backend->shaders_desired) { - if (backend->pm_current_policy && - backend->pm_current_policy->handle_event) - backend->pm_current_policy->handle_event( - kbdev, KBASE_PM_POLICY_EVENT_IDLE); - - if (kbdev->pm.backend.protected_transition_override || - (kbase_has_arbiter(kbdev) && (kbase_pm_is_suspending(kbdev) || - kbase_io_is_aw_removed(kbdev))) || - !stt->configured_ticks || WARN_ON(stt->cancel_queued)) { - backend->shaders_state = - KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - } else { - stt->remaining_ticks = stt->configured_ticks; - stt->needed = true; - - /* The shader hysteresis timer is not - * done the obvious way, which would be - * to start an hrtimer when the shader - * power off is requested. Instead, - * use a 'tick' timer, and set the - * remaining number of ticks on a power - * off request. This avoids the - * latency of starting, then - * immediately cancelling an hrtimer - * when the shaders are re-requested - * before the timeout expires. - */ - if (!hrtimer_active(&stt->timer)) - hrtimer_start(&stt->timer, stt->configured_interval, - HRTIMER_MODE_REL); - - backend->shaders_state = - KBASE_SHADERS_WAIT_OFF_CORESTACK_ON; - } - } else if (backend->shaders_desired_mask & ~shaders_ready) { - /* set cores ready but not available to - * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON - * check pass - */ - backend->shaders_avail = - (backend->shaders_desired_mask | shaders_ready); - - kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, - backend->shaders_avail & ~shaders_ready, - ACTION_PWRON); - backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; - } else if (shaders_ready & ~backend->shaders_desired_mask) { - backend->shaders_state = KBASE_SHADERS_WAIT_GPU_IDLE; - } else { - backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_WAIT_OFF_CORESTACK_ON: - if (WARN_ON(!hrtimer_active(&stt->timer))) { - stt->remaining_ticks = 0; - backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - } - - if (backend->shaders_desired) { - if (backend->pm_current_policy && - backend->pm_current_policy->handle_event) - backend->pm_current_policy->handle_event( - kbdev, KBASE_PM_POLICY_EVENT_TIMER_HIT); - - stt->remaining_ticks = 0; - backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK; - } else if (stt->remaining_ticks == 0) { - if (backend->pm_current_policy && - backend->pm_current_policy->handle_event) - backend->pm_current_policy->handle_event( - kbdev, KBASE_PM_POLICY_EVENT_TIMER_MISS); - - backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - } else if (kbase_has_arbiter(kbdev) && (kbase_pm_is_suspending(kbdev) || - kbase_io_is_aw_removed(kbdev))) { - backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_WAIT_GPU_IDLE: - /* If partial shader core off need to wait the job in - * running and next register finished then flush L2 - * or it might hit GPU2017-861 - */ - if (!kbase_gpu_atoms_submitted_any(kbdev)) { - backend->partial_shaderoff = true; - backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON: - if (!backend->partial_shaderoff) - shader_poweroff_timer_queue_cancel(kbdev); - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) { - kbase_gpu_start_cache_clean_nolock(kbdev, - GPU_COMMAND_CACHE_CLN_INV_L2); - backend->shaders_state = KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON; - } else { - backend->shaders_state = KBASE_SHADERS_READY_OFF_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON: - if (!kbdev->cache_clean_in_progress) - backend->shaders_state = KBASE_SHADERS_READY_OFF_CORESTACK_ON; - - break; - - case KBASE_SHADERS_READY_OFF_CORESTACK_ON: - if (backend->partial_shaderoff) { - backend->partial_shaderoff = false; - /* remove cores available but not ready to - * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON - * check pass - */ - - /* shaders_desired_mask shall be a subset of - * shaders_ready - */ - WARN_ON(backend->shaders_desired_mask & ~shaders_ready); - WARN_ON(!(backend->shaders_desired_mask & shaders_ready)); - - backend->shaders_avail = backend->shaders_desired_mask; - kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, - shaders_ready & ~backend->shaders_avail, - ACTION_PWROFF); - backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; - KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, - (shaders_ready & ~backend->shaders_avail)); - } else { - kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, shaders_ready, - ACTION_PWROFF); - - KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u); - - backend->shaders_state = KBASE_SHADERS_PEND_OFF_CORESTACK_ON; - } - break; - - case KBASE_SHADERS_PEND_OFF_CORESTACK_ON: - if (!shaders_trans && !shaders_ready) { - if (corestack_driver_control) - kbase_pm_invoke(kbdev, KBASE_PM_CORE_STACK, stacks_avail, - ACTION_PWROFF); - - backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_PEND_OFF; - } - break; - - case KBASE_SHADERS_OFF_CORESTACK_PEND_OFF: - if (!stacks_trans && !stacks_ready) { - /* On powered off, re-enable the hwcnt */ - backend->pm_shaders_core_mask = 0; - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { -#if MALI_USE_CSF - unsigned long flags; - - kbase_csf_scheduler_spin_lock(kbdev, &flags); -#endif - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -#if MALI_USE_CSF - kbase_csf_scheduler_spin_unlock(kbdev, flags); -#endif - backend->hwcnt_disabled = false; - } - backend->shaders_state = - KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF; - } - break; - - case KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF: - if (!hrtimer_active(&stt->timer) && !stt->cancel_queued) - backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; - break; - - case KBASE_SHADERS_RESET_WAIT: - /* Reset complete */ - if (!backend->in_reset) - backend->shaders_state = - KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF; - break; - } - - if (backend->shaders_state != prev_state) { - dev_dbg(kbdev->dev, "Shader state transition: %s to %s\n", - kbase_shader_core_state_to_string(prev_state), - kbase_shader_core_state_to_string(backend->shaders_state)); - KBASE_TLSTREAM_TL_KBASE_DEVICE_SHADER_CORE_STATE(kbdev, kbdev->id, - backend->shaders_state); - } - - } while (backend->shaders_state != prev_state); - - return 0; -} -#endif /* !MALI_USE_CSF */ - static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev) { bool in_desired_state = true; @@ -3018,16 +2416,7 @@ static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev) in_desired_state = kbase_pm_l2_is_in_desired_state(kbdev); -#if !MALI_USE_CSF - if (kbdev->pm.backend.shaders_desired && - kbdev->pm.backend.shaders_state != KBASE_SHADERS_ON_CORESTACK_ON) - in_desired_state = false; - else if (!kbdev->pm.backend.shaders_desired && - kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) - in_desired_state = false; -#else in_desired_state &= kbase_pm_mcu_is_in_desired_state(kbdev); -#endif return in_desired_state; } @@ -3058,31 +2447,9 @@ static bool kbase_pm_is_in_desired_state_with_l2_powered(struct kbase_device *kb return in_desired_state; } -#if !MALI_USE_CSF -static void kbase_pm_trace_power_state(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - KBASE_TLSTREAM_AUX_PM_STATE(kbdev, TL_PM_STATE_L2, - kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2)); - KBASE_TLSTREAM_AUX_PM_STATE(kbdev, TL_PM_STATE_SHADER, - kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER)); - KBASE_TLSTREAM_AUX_PM_STATE(kbdev, TL_PM_STATE_TILER, - kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER)); - - if (corestack_driver_control) - KBASE_TLSTREAM_AUX_PM_STATE(kbdev, TL_PM_STATE_STACK, - kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK)); -} -#endif - void kbase_pm_update_state(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - enum kbase_shader_core_state prev_shaders_state = kbdev->pm.backend.shaders_state; -#else enum kbase_mcu_state prev_mcu_state = kbdev->pm.backend.mcu_state; -#endif lockdep_assert_held(&kbdev->hwaccess_lock); @@ -3092,20 +2459,6 @@ void kbase_pm_update_state(struct kbase_device *kbdev) if (kbase_pm_l2_update_state(kbdev)) return; -#if !MALI_USE_CSF - if (kbase_pm_shaders_update_state(kbdev)) - return; - - /* If the shaders just turned off, re-invoke the L2 state machine, in - * case it was waiting for the shaders to turn off before powering down - * the L2. - */ - if (prev_shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF && - kbdev->pm.backend.shaders_state == KBASE_SHADERS_OFF_CORESTACK_OFF) { - if (kbase_pm_l2_update_state(kbdev)) - return; - } -#else if (kbase_pm_mcu_update_state(kbdev)) return; @@ -3114,15 +2467,10 @@ void kbase_pm_update_state(struct kbase_device *kbdev) if (kbase_pm_l2_update_state(kbdev)) return; } -#endif if (kbase_pm_is_in_desired_state_nolock(kbdev)) { KBASE_KTRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, kbdev->pm.backend.shaders_avail); -#if !MALI_USE_CSF - kbase_pm_trace_power_state(kbdev); -#endif - KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0); wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); } @@ -3177,7 +2525,6 @@ int kbase_pm_state_machine_init(struct kbase_device *kbdev) stt->default_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; stt->configured_ticks = stt->default_ticks; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { const u64 neural_present = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_NEURAL); const u64 shader_present = kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER); @@ -3216,16 +2563,13 @@ int kbase_pm_state_machine_init(struct kbase_device *kbdev) } INIT_WORK(&kbdev->pm.backend.core_idle_work, core_idle_worker); -#endif return 0; } void kbase_pm_state_machine_term(struct kbase_device *kbdev) { -#if MALI_USE_CSF destroy_workqueue(kbdev->pm.backend.core_idle_wq); -#endif hrtimer_cancel(&kbdev->pm.backend.shader_tick_timer.timer); destroy_workqueue(kbdev->pm.backend.shader_tick_timer.wq); } @@ -3239,23 +2583,19 @@ void kbase_pm_reset_start_locked(struct kbase_device *kbdev) backend->in_reset = true; backend->l2_state = KBASE_L2_RESET_WAIT; KBASE_KTRACE_ADD(kbdev, PM_L2_RESET_WAIT, NULL, backend->l2_state); -#if !MALI_USE_CSF - backend->shaders_state = KBASE_SHADERS_RESET_WAIT; -#else /* MCU state machine is exercised only after the initial load/boot * of the firmware. */ if (likely(kbdev->csf.firmware_inited)) { backend->mcu_state = KBASE_MCU_RESET_WAIT; KBASE_KTRACE_ADD(kbdev, PM_MCU_RESET_WAIT, NULL, backend->mcu_state); -#ifdef KBASE_PM_RUNTIME - backend->exit_gpu_sleep_mode = true; -#endif + if (IS_ENABLED(CONFIG_PM)) + backend->exit_gpu_sleep_mode = true; + kbdev->csf.firmware_reload_needed = true; } else { WARN_ON(backend->mcu_state != KBASE_MCU_OFF); } -#endif /* We're in a reset, so hwcnt will have been synchronously disabled by * this function's caller as part of the reset process. We therefore @@ -3288,42 +2628,39 @@ void kbase_pm_reset_complete(struct kbase_device *kbdev) */ kbase_gpu_cache_clean_wait_complete(kbdev); backend->in_reset = false; -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - backend->gpu_wakeup_override = false; - backend->db_mirror_interrupt_enabled = false; - backend->gpu_sleep_mode_active = false; - backend->exit_gpu_sleep_mode = false; -#endif + if (IS_ENABLED(CONFIG_PM)) { + backend->gpu_wakeup_override = false; + backend->db_mirror_interrupt_enabled = false; + backend->gpu_sleep_mode_active = false; + backend->exit_gpu_sleep_mode = false; + } kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -#if !MALI_USE_CSF -/* Timeout in milliseconds for GPU Power Management to reach the desired - * Shader and L2 state. If the time spent waiting has exceeded this threshold - * then there is most likely a hardware issue. - */ -#define PM_TIMEOUT_MS (5000) /* 5s */ -#endif +#if MALI_UNIT_TEST +void kbase_pm_timed_out(struct kbase_device *kbdev, const char *timeout_msg); +void kbase_pm_timed_out(struct kbase_device *kbdev, const char *timeout_msg) +#else static void kbase_pm_timed_out(struct kbase_device *kbdev, const char *timeout_msg) +#endif { unsigned long flags; dev_err(kbdev->dev, "%s", timeout_msg); -#if !MALI_USE_CSF - CSTD_UNUSED(flags); - dev_err(kbdev->dev, "Desired state :\n"); - dev_err(kbdev->dev, "\tShader=%016llx\n", - kbdev->pm.backend.shaders_desired ? kbdev->pm.backend.shaders_avail : 0); -#else spin_lock_irqsave(&kbdev->hwaccess_lock, flags); dev_err(kbdev->dev, "\tMCU desired = %d\n", kbase_pm_is_mcu_desired(kbdev)); dev_err(kbdev->dev, "\tMCU sw state = %d\n", kbdev->pm.backend.mcu_state); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#endif dev_err(kbdev->dev, "Current state :\n"); + + if (kbase_io_is_aw_removed(kbdev)) { + dev_err(kbdev->dev, "\tGPU lost"); + return; + } + dev_err(kbdev->dev, "\tStack =%016llx\n", kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(STACK_READY))); dev_err(kbdev->dev, "\tShader=%016llx\n", @@ -3332,10 +2669,8 @@ static void kbase_pm_timed_out(struct kbase_device *kbdev, const char *timeout_m kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(TILER_READY))); dev_err(kbdev->dev, "\tL2 =%016llx\n", kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(L2_READY))); -#if MALI_USE_CSF dev_err(kbdev->dev, "\tMCU status = %d\n", kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_STATUS))); -#endif dev_err(kbdev->dev, "Cores transitioning :\n"); dev_err(kbdev->dev, "\tStack =%016llx\n", kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(STACK_PWRTRANS))); @@ -3351,6 +2686,9 @@ static void kbase_pm_timed_out(struct kbase_device *kbdev, const char *timeout_m kbase_reset_gpu(kbdev); } } +#if MALI_UNIT_TEST +KBASE_EXPORT_TEST_API(kbase_pm_timed_out); +#endif int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev) { @@ -3363,12 +2701,8 @@ int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev) kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF timeout = (unsigned long)kbase_csf_timeout_in_jiffies( kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT)); -#else - timeout = msecs_to_jiffies(PM_TIMEOUT_MS); -#endif /* Wait for cores */ #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE @@ -3396,11 +2730,7 @@ static int pm_wait_for_desired_state(struct kbase_device *kbdev, bool killable_w { unsigned long flags; long remaining; -#if MALI_USE_CSF long timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT)); -#else - long timeout = (long)msecs_to_jiffies(PM_TIMEOUT_MS); -#endif int err = 0; /* Let the state machine latch the most recent desired state. */ @@ -3412,14 +2742,17 @@ static int pm_wait_for_desired_state(struct kbase_device *kbdev, bool killable_w #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE if (killable_wait) remaining = wait_event_killable_timeout(kbdev->pm.backend.gpu_in_desired_state_wait, - kbase_pm_is_in_desired_state(kbdev), + kbase_pm_is_in_desired_state(kbdev) || + kbase_io_is_aw_removed(kbdev), timeout); #else killable_wait = false; #endif if (!killable_wait) remaining = wait_event_timeout(kbdev->pm.backend.gpu_in_desired_state_wait, - kbase_pm_is_in_desired_state(kbdev), timeout); + kbase_pm_is_in_desired_state(kbdev) || + kbase_io_is_aw_removed(kbdev), + timeout); if (!remaining) { kbase_pm_timed_out(kbdev, "Wait for power transition timed out"); err = -ETIMEDOUT; @@ -3427,6 +2760,9 @@ static int pm_wait_for_desired_state(struct kbase_device *kbdev, bool killable_w WARN_ON_ONCE(!killable_wait); dev_info(kbdev->dev, "Wait for power transition got interrupted"); err = (int)remaining; + } else if (kbase_io_is_aw_removed(kbdev) && !kbase_pm_is_in_desired_state(kbdev)) { + dev_warn(kbdev->dev, "%s(): aborting, AW is no longer connected", __func__); + err = -ETIMEDOUT; } return err; @@ -3443,7 +2779,6 @@ int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev) } KBASE_EXPORT_TEST_API(kbase_pm_wait_for_desired_state); -#if MALI_USE_CSF /** * core_mask_update_done - Check if downscaling of shader cores is done * @@ -3497,7 +2832,6 @@ int kbase_pm_wait_for_cores_down_scale(struct kbase_device *kbdev) return err; } -#endif static bool is_poweroff_wait_in_progress(struct kbase_device *kbdev) { @@ -3515,7 +2849,6 @@ static int pm_wait_for_poweroff_work_complete(struct kbase_device *kbdev, bool k { long remaining; /* Indefinite wait needed when arbitration is enabled */ -#if MALI_USE_CSF /* gpu_poweroff_wait_work would be subjected to the kernel scheduling * and so the wait time can't only be the function of GPU frequency. */ @@ -3524,16 +2857,22 @@ static int pm_wait_for_poweroff_work_complete(struct kbase_device *kbdev, bool k kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT) + extra_wait_time_ms); const long timeout = kbase_has_arbiter(kbdev) ? MAX_SCHEDULE_TIMEOUT : timeout_extra_wait_time; -#else - const long timeout = kbase_has_arbiter(kbdev) ? MAX_SCHEDULE_TIMEOUT : - (long)msecs_to_jiffies(PM_TIMEOUT_MS); -#endif int err = 0; + unsigned long flags; + + if (kbase_io_is_aw_removed(kbdev)) { + dev_info(kbdev->dev, "%s: AW disconnected, exiting early", __func__); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbdev->pm.backend.poweroff_wait_in_progress = false; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return err; + } #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE if (killable_wait) remaining = wait_event_killable_timeout(kbdev->pm.backend.poweroff_wait, - !is_poweroff_wait_in_progress(kbdev), + !is_poweroff_wait_in_progress(kbdev) || + kbase_io_is_aw_removed(kbdev), timeout); #else killable_wait = false; @@ -3541,7 +2880,8 @@ static int pm_wait_for_poweroff_work_complete(struct kbase_device *kbdev, bool k if (!killable_wait) remaining = wait_event_timeout(kbdev->pm.backend.poweroff_wait, - !is_poweroff_wait_in_progress(kbdev), timeout); + !is_poweroff_wait_in_progress(kbdev), timeout) || + kbase_io_is_aw_removed(kbdev); if (!remaining) { kbase_pm_timed_out(kbdev, "Wait for poweroff work timed out"); err = -ETIMEDOUT; @@ -3571,11 +2911,7 @@ void kbase_pm_enable_interrupts(struct kbase_device *kbdev) bool is_legacy_gpu_irq_mask; KBASE_DEBUG_ASSERT(kbdev != NULL); -#if MALI_USE_CSF is_legacy_gpu_irq_mask = !kbdev->pm.backend.has_host_pwr_iface; -#else - is_legacy_gpu_irq_mask = true; -#endif /* * Clear all interrupts, @@ -3593,19 +2929,13 @@ void kbase_pm_enable_interrupts(struct kbase_device *kbdev) kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK), 0xFFFFFFFF); kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_CLEAR), 0xFFFFFFFF); -#if MALI_USE_CSF /* Enable only the Page fault bits part */ kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), 0xFFFF); -#else - kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), 0xFFFFFFFF); -#endif -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_CLEAR), PWR_IRQ_REG_ALL); kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_MASK), PWR_IRQ_REG_ALL); } -#endif } KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); @@ -3615,11 +2945,7 @@ void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) bool is_legacy_gpu_irq_mask; KBASE_DEBUG_ASSERT(kbdev != NULL); -#if MALI_USE_CSF is_legacy_gpu_irq_mask = !kbdev->pm.backend.has_host_pwr_iface; -#else - is_legacy_gpu_irq_mask = true; -#endif /* * Mask all interrupts, * and clear them all. @@ -3636,12 +2962,10 @@ void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), 0); kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_CLEAR), 0xFFFFFFFF); -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_MASK), 0); kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_IRQ_CLEAR), PWR_IRQ_REG_ALL); } -#endif } void kbase_pm_disable_interrupts(struct kbase_device *kbdev) @@ -3655,7 +2979,6 @@ void kbase_pm_disable_interrupts(struct kbase_device *kbdev) KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts); -#if MALI_USE_CSF /** * update_user_reg_page_mapping - Update the mapping for USER Register page * @@ -3686,7 +3009,6 @@ static void update_user_reg_page_mapping(struct kbase_device *kbdev) } mutex_unlock(&kbdev->csf.reg_lock); } -#endif /* * pmu layout: @@ -3701,9 +3023,6 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) unsigned long flags; KBASE_DEBUG_ASSERT(kbdev != NULL); -#if !MALI_USE_CSF - lockdep_assert_held(&kbdev->js_data.runpool_mutex); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kbdev->pm.lock); if (WARN_ON(kbase_io_is_aw_removed(kbdev))) { @@ -3712,12 +3031,11 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) } if (kbase_io_is_gpu_powered(kbdev)) { -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - if (backend->gpu_idled) { + if (IS_ENABLED(CONFIG_PM) && backend->gpu_idled) { backend->callback_power_runtime_gpu_active(kbdev); backend->gpu_idled = false; } -#endif + /* Already turned on */ if (kbdev->poweroff_pending) kbase_pm_enable_interrupts(kbdev); @@ -3741,10 +3059,8 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_GPU_OFF); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF /* GPU has been turned on, can switch to actual register page */ update_user_reg_page_mapping(kbdev); -#endif if (ret == GPU_STATE_IN_RESET) { @@ -3783,13 +3099,6 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); mutex_unlock(&kbdev->mmu_hw_mutex); - if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) { - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_dummy_job_wa_execute(kbdev, - kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER)); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } - /* Enable the interrupts */ kbase_pm_enable_interrupts(kbdev); @@ -3797,7 +3106,6 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); backend->gpu_ready = true; backend->l2_desired = true; -#if MALI_USE_CSF { if (ret != GPU_STATE_INTACT) { /* GPU reset was done after the power on, so send the post @@ -3809,19 +3117,16 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) kbase_ipa_control_handle_gpu_power_on(kbdev); } } -#endif kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) /* GPU is now powered up. Invoke the GPU active callback as GPU idle * callback would have been invoked before the power down. */ - if (backend->gpu_idled) { + if (IS_ENABLED(CONFIG_PM) && backend->gpu_idled) { backend->callback_power_runtime_gpu_active(kbdev); backend->gpu_idled = false; } -#endif } KBASE_EXPORT_TEST_API(kbase_pm_clock_on); @@ -3861,11 +3166,9 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) } kbase_pm_cache_snoop_disable(kbdev); -#if MALI_USE_CSF { kbase_ipa_control_handle_gpu_power_off(kbdev); } -#endif if (kbase_io_is_aw_removed(kbdev)) { /* Ensure we unblock any threads that are stuck waiting @@ -3881,10 +3184,8 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF /* GPU is about to be turned off, switch to dummy page */ update_user_reg_page_mapping(kbdev); -#endif kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT); if (kbdev->pm.backend.callback_power_off) @@ -3940,7 +3241,6 @@ static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) return HRTIMER_NORESTART; } -#if MALI_USE_CSF /** * kbase_ne_control_apply - Apply DT values to NEURAL_CONTROL register * @@ -3979,37 +3279,11 @@ static int kbase_ne_control_apply(struct kbase_device *kbdev) return 0; } -#endif static int kbase_set_gpu_quirks(struct kbase_device *kbdev) { -#if MALI_USE_CSF if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(CSF_CONFIG))) kbdev->hw_quirks_gpu = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(CSF_CONFIG)); -#else - u32 hw_quirks_gpu = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(JM_CONFIG)); - - if (kbdev->gpu_props.gpu_id.product_model == GPU_ID_PRODUCT_TMIX) { - /* Only for tMIx */ - u32 coherency_features; - - coherency_features = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(COHERENCY_FEATURES)); - - /* (COHERENCY_ACE_LITE | COHERENCY_ACE) was incorrectly - * documented for tMIx so force correct value here. - */ - if (coherency_features == COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { - hw_quirks_gpu |= (COHERENCY_ACE_LITE | COHERENCY_ACE) - << JM_FORCE_COHERENCY_FEATURES_SHIFT; - } - } - - if (!kbase_io_has_gpu(kbdev)) - return -EIO; - - kbdev->hw_quirks_gpu = hw_quirks_gpu; - -#endif /* !MALI_USE_CSF */ if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_IDVS_GROUP_SIZE)) { u32 default_idvs_group_size = 0xF; u32 group_size = 0; @@ -4074,7 +3348,6 @@ static int kbase_set_tiler_quirks(struct kbase_device *kbdev) return 0; } -#if MALI_USE_CSF static int kbase_set_ne_quirks(struct kbase_device *kbdev) { u32 hw_quirks_ne = 0; @@ -4089,7 +3362,6 @@ static int kbase_set_ne_quirks(struct kbase_device *kbdev) return 0; } -#endif static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) { @@ -4143,7 +3415,6 @@ static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) error = kbase_set_mmu_quirks(kbdev); } -#if MALI_USE_CSF if (!of_property_read_u32(np, "quirks-ne", &kbdev->hw_quirks_ne)) { dev_info(kbdev->dev, "Found NE quirks = [0x%x] in Devicetree\n", kbdev->hw_quirks_ne); @@ -4152,7 +3423,6 @@ static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) if (error) return error; } -#endif return error; } @@ -4167,64 +3437,21 @@ static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(L2_MMU_CONFIG))) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(L2_MMU_CONFIG), kbdev->hw_quirks_mmu); -#if MALI_USE_CSF if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(CSF_CONFIG))) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(CSF_CONFIG), kbdev->hw_quirks_gpu); -#else - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(JM_CONFIG), kbdev->hw_quirks_gpu); -#endif -#if MALI_USE_CSF if (kbase_reg_is_valid(kbdev, GPU_CONTROL_ENUM(NEURAL_CONFIG))) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(NEURAL_CONFIG), kbdev->hw_quirks_ne); -#endif } void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && !kbdev->cci_snoop_enabled) { -#if IS_ENABLED(CONFIG_ARM64) - if (kbdev->snoop_enable_smc != 0) - kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0); -#endif /* CONFIG_ARM64 */ - dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n"); - kbdev->cci_snoop_enabled = true; - } -#endif /* !MALI_USE_CSF */ } void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - if (kbdev->cci_snoop_enabled) { -#if IS_ENABLED(CONFIG_ARM64) - if (kbdev->snoop_disable_smc != 0) { - mali_cci_flush_l2(kbdev); - kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0); - } -#endif /* CONFIG_ARM64 */ - dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n"); - kbdev->cci_snoop_enabled = false; - } -#endif /* !MALI_USE_CSF */ } -#if !MALI_USE_CSF -static void reenable_protected_mode_hwcnt(struct kbase_device *kbdev) -{ - unsigned long irq_flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); - kbdev->protected_mode_hwcnt_desired = true; - if (kbdev->protected_mode_hwcnt_disabled) { - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbdev->protected_mode_hwcnt_disabled = false; - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -} -#endif - static int kbase_pm_do_reset_soft(struct kbase_device *kbdev) { int ret; @@ -4236,14 +3463,11 @@ static int kbase_pm_do_reset_soft(struct kbase_device *kbdev) else if (ret > 0) return 0; } else { -#if MALI_USE_CSF /* Check if soft reset needs to be issued via PWR_COMMAND */ if (kbdev->pm.backend.has_host_pwr_iface) kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_COMMAND), PWR_COMMAND_COMMAND_RESET_SOFT); - else -#endif - { + else { kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_SOFT_RESET); } @@ -4272,12 +3496,10 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) reg_val = RESET_COMPLETED; /* Unmask the reset complete interrupt only */ -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { reg_offset = HOST_POWER_ENUM(PWR_IRQ_MASK); reg_val = PWR_IRQ_RESET_COMPLETED; } -#endif kbase_reg_write32(kbdev, reg_offset, reg_val); } @@ -4286,14 +3508,27 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) rtdata.timed_out = false; /* Create a timer to use as a timeout on the reset */ +#if KERNEL_VERSION(6, 13, 0) <= LINUX_VERSION_CODE + hrtimer_setup_on_stack(&rtdata.timer, kbasep_reset_timeout, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); +#else hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); rtdata.timer.function = kbasep_reset_timeout; +#endif hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), HRTIMER_MODE_REL); /* Wait for the RESET_COMPLETED interrupt to be raised */ kbase_pm_wait_for_reset(kbdev); + if (kbase_io_is_aw_removed(kbdev)) { + dev_warn(kbdev->dev, "%s: AW removed, exiting early", __func__); + hrtimer_cancel(&rtdata.timer); + destroy_hrtimer_on_stack(&rtdata.timer); + + return -ENODEV; + } + if (!rtdata.timed_out) { /* GPU has been reset */ @@ -4305,12 +3540,10 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) reg_offset = GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT); reg_val = RESET_COMPLETED; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { reg_offset = HOST_POWER_ENUM(PWR_IRQ_RAWSTAT); reg_val = PWR_IRQ_RESET_COMPLETED; } -#endif /* No interrupt has been received - check if the RAWSTAT register says * the reset has completed. @@ -4344,13 +3577,10 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) atomic_set(&kbdev->pm.backend.reset_in_progress, 1); -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) kbase_reg_write32(kbdev, HOST_POWER_ENUM(PWR_COMMAND), PWR_COMMAND_COMMAND_RESET_HARD); - else -#endif - { + else { kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_HARD_RESET); } @@ -4408,10 +3638,8 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_GPU_OFF); } -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface && !pwr_status_gpu_reset_allowed(kbdev)) return -EINVAL; -#endif /* Ensure interrupts are off to begin with, this also clears any * outstanding interrupts @@ -4434,7 +3662,6 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) err = kbdev->protected_ops->protected_mode_disable(kbdev->protected_dev); spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); -#if MALI_USE_CSF if (kbdev->protected_mode) { unsigned long flags; @@ -4444,14 +3671,12 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_hwcnt_backend_csf_protm_exited(&kbdev->hwcnt_gpu_iface); kbase_csf_scheduler_spin_unlock(kbdev, flags); } -#endif kbdev->protected_mode = false; spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); if (err) goto exit; -#if MALI_USE_CSF /* Check power status register value after GPU reset */ if (kbdev->pm.backend.has_host_pwr_iface && !pwr_status_value_valid_post_reset(kbdev)) { err = -EINVAL; @@ -4469,7 +3694,6 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) if (err) goto exit; } -#endif if (flags & PM_HW_ISSUES_DETECT) { err = kbase_pm_hw_issues_detect(kbdev); if (err) @@ -4479,10 +3703,8 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_pm_hw_issues_apply(kbdev); kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency); kbase_amba_set_shareable_cache_support(kbdev); -#if MALI_USE_CSF kbase_backend_update_gpu_timestamp_offset(kbdev); kbdev->csf.compute_progress_timeout_cc = 0; -#endif /* Sanity check protected mode was left after reset */ WARN_ON(kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) & @@ -4492,14 +3714,6 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_pm_enable_interrupts(kbdev); exit: -#if !MALI_USE_CSF - if (!kbdev->pm.backend.protected_entry_transition_override) { - /* Re-enable GPU hardware counters if we're resetting from - * protected mode. - */ - reenable_protected_mode_hwcnt(kbdev); - } -#endif return err; } @@ -4532,13 +3746,6 @@ static void kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *k /* This might happen after GPU reset. * Then counter needs to be kicked. */ -#if !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) && !MALI_USE_CSF - if (!(kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) & - GPU_STATUS_CYCLE_COUNT_ACTIVE)) { - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), - GPU_COMMAND_CYCLE_COUNT_START); - } -#endif } spin_unlock_irqrestore(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, flags); diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_internal.h index b6e0311308e1..88ae28192851 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_internal.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_internal.h @@ -31,6 +31,19 @@ #include "backend/gpu/mali_kbase_pm_ca.h" #include "mali_kbase_pm_policy.h" +/** + * enum mask_type - used to determine which mask is being updated + * + * @SYSFS_COREMASK: core mask requested via sysfs + * @DEVFREQ_COREMASK: core mask requested via devfreq + */ +enum mask_type { + SYSFS_COREMASK, +#ifdef CONFIG_MALI_VALHALL_DEVFREQ + DEVFREQ_COREMASK +#endif +}; + /** * kbase_pm_dev_idle - The GPU is idle. * @@ -200,7 +213,6 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags); */ void kbase_pm_reset_done(struct kbase_device *kbdev); -#if MALI_USE_CSF /** * kbase_pm_wait_for_desired_state - Wait for the desired power state to be * reached @@ -222,30 +234,6 @@ void kbase_pm_reset_done(struct kbase_device *kbdev); * Return: 0 on success, or -ETIMEDOUT code on timeout error. */ int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); -#else -/** - * kbase_pm_wait_for_desired_state - Wait for the desired power state to be - * reached - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Wait for the L2 and shader power state machines to reach the states - * corresponding to the values of 'l2_desired' and 'shaders_desired'. - * - * The usual use-case for this is to ensure cores are 'READY' after performing - * a GPU Reset. - * - * Unlike kbase_pm_update_state(), the caller must not hold hwaccess_lock, - * because this function will take that lock itself. - * - * NOTE: This may not wait until the correct state is reached if there is a - * power off in progress. To correctly wait for the desired state the caller - * must ensure that this is not the case by, for example, calling - * kbase_pm_wait_for_poweroff_work_complete() - * - * Return: 0 on success, or -ETIMEDOUT error code on timeout error. - */ -int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); -#endif /** * kbase_pm_killable_wait_for_desired_state - Wait for the desired power state to be @@ -279,7 +267,6 @@ int kbase_pm_killable_wait_for_desired_state(struct kbase_device *kbdev); */ int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev); -#if MALI_USE_CSF /** * kbase_pm_wait_for_cores_down_scale - Wait for the downscaling of shader cores * @@ -308,7 +295,6 @@ int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev); * Return: 0 on success, error code on error or remaining jiffies on timeout. */ int kbase_pm_wait_for_cores_down_scale(struct kbase_device *kbdev); -#endif /** * kbase_pm_update_dynamic_cores_onoff - Update the L2 and shader power state @@ -534,14 +520,11 @@ void kbase_pm_runtime_term(struct kbase_device *kbdev); * @kbdev: The kbase device structure for the device (must be a valid pointer) * * Enables access to the GPU registers before power management has powered up - * the GPU with kbase_pm_powerup(). + * the GPU. * * This results in the power management callbacks provided in the driver * configuration to get called to turn on power and/or clocks to the GPU. See * kbase_pm_callback_conf. - * - * This should only be used before power management is powered up with - * kbase_pm_powerup() */ void kbase_pm_register_access_enable(struct kbase_device *kbdev); @@ -555,9 +538,6 @@ void kbase_pm_register_access_enable(struct kbase_device *kbdev); * This results in the power management callbacks provided in the driver * configuration to get called to turn off power and/or clocks to the GPU. See * kbase_pm_callback_conf - * - * This should only be used before power management is powered up with - * kbase_pm_powerup() */ void kbase_pm_register_access_disable(struct kbase_device *kbdev); @@ -606,7 +586,6 @@ void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, struct kbasep_pm_metr #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF /** * kbase_platform_dvfs_event - Report utilisation to DVFS code for CSF GPU * @@ -620,24 +599,6 @@ void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, struct kbasep_pm_metr * Return: Returns 0 on failure and non zero on success. */ int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation); -#else -/** - * kbase_platform_dvfs_event - Report utilisation to DVFS code for JM GPU - * - * @kbdev: The kbase device structure for the device (must be a - * valid pointer) - * @utilisation: The current calculated utilisation by the metrics system. - * @util_gl_share: The current calculated gl share of utilisation. - * @util_cl_share: The current calculated cl share of utilisation per core - * group. - * Function provided by platform specific code when DVFS is enabled to allow - * the power management metrics system to report utilisation. - * - * Return: Returns 0 on failure and non zero on success. - */ -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]); -#endif #endif /* CONFIG_MALI_VALHALL_DVFS */ @@ -672,6 +633,28 @@ void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev); */ void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev); +/** + * kbase_pm_ca_set_gov_core_mask - Set governor core mask + * @kbdev: Device pointer. + * @core_mask_type: which mask is being used to update register. + * @core_mask: New core mask. + * + * This function is used to change the available core mask as defined via either sysfs or devfreq. + */ +void kbase_pm_ca_set_gov_core_mask(struct kbase_device *kbdev, enum mask_type core_mask_type, + u64 core_mask); + +/** + * kbase_pm_ca_set_gov_core_mask_nolock - Set governor core mask with lock already taken + * @kbdev: Device pointer. + * @core_mask_type: which mask is being used to update register. + * @core_mask: New core mask. + * + * This function is used to change the available core mask as defined via either sysfs or devfreq. + */ +void kbase_pm_ca_set_gov_core_mask_nolock(struct kbase_device *kbdev, enum mask_type core_mask_type, + u64 core_mask); + #ifdef CONFIG_MALI_VALHALL_DEVFREQ /** * kbase_devfreq_set_core_mask - Set devfreq core mask @@ -704,81 +687,6 @@ void kbase_pm_reset_start_locked(struct kbase_device *kbdev); */ void kbase_pm_reset_complete(struct kbase_device *kbdev); -#if !MALI_USE_CSF -/** - * kbase_pm_protected_override_enable - Enable the protected mode override - * @kbdev: Device pointer - * - * When the protected mode override is enabled, all shader cores are requested - * to power down, and the L2 power state can be controlled by - * kbase_pm_protected_l2_override(). - * - * Caller must hold hwaccess_lock. - */ -void kbase_pm_protected_override_enable(struct kbase_device *kbdev); - -/** - * kbase_pm_protected_override_disable - Disable the protected mode override - * @kbdev: Device pointer - * - * Caller must hold hwaccess_lock. - */ -void kbase_pm_protected_override_disable(struct kbase_device *kbdev); - -/** - * kbase_pm_protected_l2_override - Control the protected mode L2 override - * @kbdev: Device pointer - * @override: true to enable the override, false to disable - * - * When the driver is transitioning in or out of protected mode, the L2 cache is - * forced to power off. This can be overridden to force the L2 cache to power - * on. This is required to change coherency settings on some GPUs. - */ -void kbase_pm_protected_l2_override(struct kbase_device *kbdev, bool override); - -/** - * kbase_pm_protected_entry_override_enable - Enable the protected mode entry - * override - * @kbdev: Device pointer - * - * Initiate a GPU reset and enable the protected mode entry override flag if - * l2_always_on WA is enabled and platform is fully coherent. If the GPU - * reset is already ongoing then protected mode entry override flag will not - * be enabled and function will have to be called again. - * - * When protected mode entry override flag is enabled to power down L2 via GPU - * reset, the GPU reset handling behavior gets changed. For example call to - * kbase_backend_reset() is skipped, Hw counters are not re-enabled and L2 - * isn't powered up again post reset. - * This is needed only as a workaround for a Hw issue where explicit power down - * of L2 causes a glitch. For entering protected mode on fully coherent - * platforms L2 needs to be powered down to switch to IO coherency mode, so to - * avoid the glitch GPU reset is used to power down L2. Hence, this function - * does nothing on systems where the glitch issue isn't present. - * - * Caller must hold hwaccess_lock. Should be only called during the transition - * to enter protected mode. - * - * Return: -EAGAIN if a GPU reset was required for the glitch workaround but - * was already ongoing, otherwise 0. - */ -int kbase_pm_protected_entry_override_enable(struct kbase_device *kbdev); - -/** - * kbase_pm_protected_entry_override_disable - Disable the protected mode entry - * override - * @kbdev: Device pointer - * - * This shall be called once L2 has powered down and switch to IO coherency - * mode has been made. As with kbase_pm_protected_entry_override_enable(), - * this function does nothing on systems where the glitch issue isn't present. - * - * Caller must hold hwaccess_lock. Should be only called during the transition - * to enter protected mode. - */ -void kbase_pm_protected_entry_override_disable(struct kbase_device *kbdev); -#endif - /* If true, the driver should explicitly control corestack power management, * instead of relying on the Power Domain Controller. */ @@ -795,7 +703,6 @@ extern bool corestack_driver_control; */ bool kbase_pm_is_l2_desired(struct kbase_device *kbdev); -#if MALI_USE_CSF /** * kbase_pm_is_mcu_desired - Check whether MCU is desired * @@ -821,8 +728,6 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev); */ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state state); -#ifdef KBASE_PM_RUNTIME - /** * kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on * MCU side @@ -834,8 +739,6 @@ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state s */ void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev); -#endif /* KBASE_PM_RUNTIME */ - /** * kbase_pm_idle_groups_sched_suspendable - Check whether the scheduler can be * suspended to low power state when all @@ -907,8 +810,6 @@ static inline bool kbase_pm_mcu_is_in_desired_state(struct kbase_device *kbdev) return in_desired_state; } -#endif - /** * kbase_pm_l2_is_in_desired_state - Check if L2 is in stable ON/OFF state. * @@ -937,9 +838,6 @@ static inline bool kbase_pm_l2_is_in_desired_state(struct kbase_device *kbdev) */ static inline void kbase_pm_lock(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - mutex_lock(&kbdev->js_data.runpool_mutex); -#endif /* !MALI_USE_CSF */ mutex_lock(&kbdev->pm.lock); } @@ -951,12 +849,8 @@ static inline void kbase_pm_lock(struct kbase_device *kbdev) static inline void kbase_pm_unlock(struct kbase_device *kbdev) { mutex_unlock(&kbdev->pm.lock); -#if !MALI_USE_CSF - mutex_unlock(&kbdev->js_data.runpool_mutex); -#endif /* !MALI_USE_CSF */ } -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) /** * kbase_pm_gpu_sleep_allowed - Check if the GPU is allowed to be put in sleep * @@ -978,8 +872,10 @@ static inline bool kbase_pm_gpu_sleep_allowed(struct kbase_device *kbdev) * A high positive value of autosuspend_delay can be used to keep the * GPU in sleep state for a long time. */ +#if IS_ENABLED(CONFIG_PM) if (unlikely(kbdev->dev->power.autosuspend_delay <= 0)) return false; +#endif return test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); } @@ -997,8 +893,10 @@ static inline bool kbase_pm_gpu_sleep_allowed(struct kbase_device *kbdev) */ static inline bool kbase_pm_fw_sleep_on_idle_allowed(struct kbase_device *kbdev) { +#if IS_ENABLED(CONFIG_PM) if (unlikely(kbdev->dev->power.autosuspend_delay <= 0)) return false; +#endif return kbdev->pm.backend.gpu_sleep_allowed == KBASE_GPU_FW_SLEEP_ON_IDLE_ALLOWED; } @@ -1049,7 +947,6 @@ static inline void kbase_pm_disable_db_mirror_interrupt(struct kbase_device *kbd kbdev->pm.backend.db_mirror_interrupt_enabled = false; } } -#endif /** * kbase_pm_l2_allow_mmu_page_migration - L2 state allows MMU page migration or not @@ -1074,7 +971,12 @@ static inline bool kbase_pm_l2_allow_mmu_page_migration(struct kbase_device *kbd return (backend->l2_state != KBASE_L2_PEND_ON && backend->l2_state != KBASE_L2_PEND_OFF); } -#if MALI_USE_CSF +#if MALI_UNIT_TEST +int delegate_pm_domain_control_to_fw(struct kbase_device *kbdev, u32 pm_domain); + +int retract_pm_domain_control_from_fw(struct kbase_device *kbdev, u32 pm_domain); +#endif + /** * kbase_pm_get_domain_status - get pm domain status for particular endpoint * @@ -1089,6 +991,5 @@ static inline bool kbase_pm_l2_allow_mmu_page_migration(struct kbase_device *kbd */ int kbase_pm_get_domain_status(struct kbase_device *kbdev, u32 pm_domain, u32 endpoint, u32 *domain_status); -#endif /* MALI_USE_CSF */ #endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_metrics.c index 4822e64f645f..2966a94efe65 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_metrics.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_metrics.c @@ -28,17 +28,13 @@ #include #include -#if MALI_USE_CSF #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" #include -#else -#include -#endif /* !MALI_USE_CSF */ #include #include -#if defined(CONFIG_MALI_VALHALL_DEVFREQ) || defined(CONFIG_MALI_VALHALL_DVFS) || !MALI_USE_CSF +#if defined(CONFIG_MALI_VALHALL_DEVFREQ) || defined(CONFIG_MALI_VALHALL_DVFS) /* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly * under 11s. Exceeding this will cause overflow @@ -46,10 +42,8 @@ #define KBASE_PM_TIME_SHIFT 8 #endif -#if MALI_USE_CSF /* To get the GPU_ACTIVE value in nano seconds unit */ #define GPU_ACTIVE_SCALING_FACTOR ((u64)1E9) -#endif /* * Possible state transitions @@ -101,7 +95,6 @@ static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) int kbasep_pm_metrics_init(struct kbase_device *kbdev) { -#if MALI_USE_CSF struct kbase_ipa_control_perf_counter perf_counter; int err; @@ -129,11 +122,6 @@ int kbasep_pm_metrics_init(struct kbase_device *kbdev) dev_err(kbdev->dev, "Failed to register IPA with kbase_ipa_control: err=%d", err); return -1; } -#else - KBASE_DEBUG_ASSERT(kbdev != NULL); - kbdev->pm.backend.metrics.kbdev = kbdev; - kbdev->pm.backend.metrics.time_period_start = ktime_get_raw(); -#endif spin_lock_init(&kbdev->pm.backend.metrics.lock); #ifdef CONFIG_MALI_VALHALL_DVFS @@ -144,13 +132,11 @@ int kbasep_pm_metrics_init(struct kbase_device *kbdev) kbase_pm_metrics_start(kbdev); #endif /* CONFIG_MALI_VALHALL_DVFS */ -#if MALI_USE_CSF /* The sanity check on the GPU_ACTIVE performance counter * is skipped for Juno platforms that have timing problems. */ kbdev->pm.backend.metrics.skip_gpu_active_sanity_check = (kbdev->gpu_props.impl_tech >= THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_FPGA); -#endif return 0; } @@ -167,11 +153,7 @@ void kbasep_pm_metrics_term(struct kbase_device *kbdev) hrtimer_cancel(&kbdev->pm.backend.metrics.timer); #endif /* CONFIG_MALI_VALHALL_DVFS */ -#if MALI_USE_CSF kbase_ipa_control_unregister(kbdev, kbdev->pm.backend.metrics.ipa_control_client); -#else - CSTD_UNUSED(kbdev); -#endif } KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); @@ -179,7 +161,6 @@ KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); /* caller needs to hold kbdev->pm.backend.metrics.lock before calling this * function */ -#if MALI_USE_CSF #if defined(CONFIG_MALI_VALHALL_DEVFREQ) || defined(CONFIG_MALI_VALHALL_DVFS) static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev) { @@ -284,39 +265,6 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev) kbdev->pm.backend.metrics.time_period_start = now; } #endif /* defined(CONFIG_MALI_VALHALL_DEVFREQ) || defined(CONFIG_MALI_VALHALL_DVFS) */ -#else -static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, ktime_t now) -{ - ktime_t diff; - - lockdep_assert_held(&kbdev->pm.backend.metrics.lock); - - diff = ktime_sub(now, kbdev->pm.backend.metrics.time_period_start); - if (ktime_to_ns(diff) < 0) - return; - - if (kbdev->pm.backend.metrics.gpu_active) { - u32 ns_time = (u32)(ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); - - kbdev->pm.backend.metrics.values.time_busy += ns_time; - if (kbdev->pm.backend.metrics.active_cl_ctx[0]) - kbdev->pm.backend.metrics.values.busy_cl[0] += ns_time; - if (kbdev->pm.backend.metrics.active_cl_ctx[1]) - kbdev->pm.backend.metrics.values.busy_cl[1] += ns_time; - if (kbdev->pm.backend.metrics.active_gl_ctx[0]) - kbdev->pm.backend.metrics.values.busy_gl += ns_time; - if (kbdev->pm.backend.metrics.active_gl_ctx[1]) - kbdev->pm.backend.metrics.values.busy_gl += ns_time; - if (kbdev->pm.backend.metrics.active_gl_ctx[2]) - kbdev->pm.backend.metrics.values.busy_gl += ns_time; - } else { - kbdev->pm.backend.metrics.values.time_idle += - (u32)(ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); - } - - kbdev->pm.backend.metrics.time_period_start = now; -} -#endif /* MALI_USE_CSF */ #if defined(CONFIG_MALI_VALHALL_DEVFREQ) || defined(CONFIG_MALI_VALHALL_DVFS) void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, struct kbasep_pm_metrics *last, @@ -326,23 +274,13 @@ void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, struct kbasep_pm_metr unsigned long flags; spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -#if MALI_USE_CSF kbase_pm_get_dvfs_utilisation_calc(kbdev); -#else - kbase_pm_get_dvfs_utilisation_calc(kbdev, ktime_get_raw()); -#endif memset(diff, 0, sizeof(*diff)); diff->time_busy = cur->time_busy - last->time_busy; diff->time_idle = cur->time_idle - last->time_idle; -#if MALI_USE_CSF diff->time_in_protm = cur->time_in_protm - last->time_in_protm; -#else - diff->busy_cl[0] = cur->busy_cl[0] - last->busy_cl[0]; - diff->busy_cl[1] = cur->busy_cl[1] - last->busy_cl[1]; - diff->busy_gl = cur->busy_gl - last->busy_gl; -#endif *last = *cur; @@ -356,11 +294,6 @@ void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) { int utilisation; struct kbasep_pm_metrics *diff; -#if !MALI_USE_CSF - int busy; - int util_gl_share; - int util_cl_share[2]; -#endif KBASE_DEBUG_ASSERT(kbdev != NULL); @@ -370,15 +303,6 @@ void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) utilisation = (100 * diff->time_busy) / max(diff->time_busy + diff->time_idle, 1u); -#if !MALI_USE_CSF - busy = max(diff->busy_gl + diff->busy_cl[0] + diff->busy_cl[1], 1u); - - util_gl_share = (100 * diff->busy_gl) / busy; - util_cl_share[0] = (100 * diff->busy_cl[0]) / busy; - util_cl_share[1] = (100 * diff->busy_cl[1]) / busy; - - kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, util_cl_share); -#else /* Note that, at present, we don't pass protected-mode time to the * platform here. It's unlikely to be useful, however, as the platform * probably just cares whether the GPU is busy or not; time in @@ -386,7 +310,6 @@ void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) * so we should be good. */ kbase_platform_dvfs_event(kbdev, utilisation); -#endif } bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) @@ -421,77 +344,3 @@ void kbase_pm_metrics_stop(struct kbase_device *kbdev) } #endif /* CONFIG_MALI_VALHALL_DVFS */ - -#if !MALI_USE_CSF -/** - * kbase_pm_metrics_active_calc - Update PM active counts based on currently - * running atoms - * @kbdev: Device pointer - * - * The caller must hold kbdev->pm.backend.metrics.lock - */ -static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) -{ - unsigned int js; - - lockdep_assert_held(&kbdev->pm.backend.metrics.lock); - - kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; - kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; - kbdev->pm.backend.metrics.active_gl_ctx[2] = 0; - kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; - kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; - kbdev->pm.backend.metrics.gpu_active = false; - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); - - /* Head atom may have just completed, so if it isn't running - * then try the next atom - */ - if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) - katom = kbase_gpu_inspect(kbdev, js, 1); - - if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) { - if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { - u32 device_nr = - (katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) ? - katom->device_nr : - 0; - if (!WARN_ON(device_nr >= 2)) - kbdev->pm.backend.metrics.active_cl_ctx[device_nr] = 1; - } else { - kbdev->pm.backend.metrics.active_gl_ctx[js] = 1; - trace_sysgraph(SGR_ACTIVE, 0, js); - } - kbdev->pm.backend.metrics.gpu_active = true; - } else { - trace_sysgraph(SGR_INACTIVE, 0, js); - } - } -} - -/* called when job is submitted to or removed from a GPU slot */ -void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) -{ - unsigned long flags; - ktime_t now; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); - - if (!timestamp) { - now = ktime_get_raw(); - timestamp = &now; - } - - /* Track how much of time has been spent busy or idle. For JM GPUs, - * this also evaluates how long CL and/or GL jobs have been busy for. - */ - kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); - - kbase_pm_metrics_active_calc(kbdev); - spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -} -#endif /* !MALI_USE_CSF */ diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_policy.c index 4ad6f703ad67..78e3ebef0ecc 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_policy.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_policy.c @@ -30,7 +30,7 @@ #include #include -#if MALI_USE_CSF && defined CONFIG_MALI_VALHALL_DEBUG +#if defined CONFIG_MALI_VALHALL_DEBUG #include #endif @@ -64,7 +64,7 @@ void kbase_pm_policy_init(struct kbase_device *kbdev) } } -#if MALI_USE_CSF && defined(CONFIG_MALI_VALHALL_DEBUG) +#if defined(CONFIG_MALI_VALHALL_DEBUG) /* Use always_on policy if module param fw_debug=1 is * passed, to aid firmware debugging. */ @@ -74,24 +74,19 @@ void kbase_pm_policy_init(struct kbase_device *kbdev) default_policy->init(kbdev); -#if MALI_USE_CSF spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->pm.backend.pm_current_policy = default_policy; kbdev->pm.backend.csf_pm_sched_flags = default_policy->pm_sched_flags; -#ifdef KBASE_PM_RUNTIME - if (kbase_pm_idle_groups_sched_suspendable(kbdev)) - clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); - else - set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); -#endif /* KBASE_PM_RUNTIME */ + if (IS_ENABLED(CONFIG_PM)) { + if (kbase_pm_idle_groups_sched_suspendable(kbdev)) + clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, + &kbdev->pm.backend.gpu_sleep_allowed); + else + set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -#else - CSTD_UNUSED(flags); - kbdev->pm.backend.pm_current_policy = default_policy; -#endif } void kbase_pm_policy_term(struct kbase_device *kbdev) @@ -123,7 +118,6 @@ void kbase_pm_update_active(struct kbase_device *kbdev) if (!pm->backend.invoke_poweroff_wait_wq_when_l2_off && pm->backend.poweroff_wait_in_progress) { KBASE_DEBUG_ASSERT(kbase_io_is_gpu_powered(kbdev)); -#if MALI_USE_CSF if (likely(!pm->backend.waiting_for_mmu_fault_handling)) { /* L2 has been powered off. Invoke the state machine to power * up the L2 cache and also effectively cancel the GPU power off @@ -137,7 +131,6 @@ void kbase_pm_update_active(struct kbase_device *kbdev) wake_up(&kbdev->pm.backend.poweroff_wait); return; } -#endif pm->backend.poweron_required = true; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } else { @@ -151,9 +144,7 @@ void kbase_pm_update_active(struct kbase_device *kbdev) pm->backend.invoke_poweroff_wait_wq_when_l2_off = false; pm->backend.poweroff_wait_in_progress = false; pm->backend.l2_desired = true; -#if MALI_USE_CSF pm->backend.mcu_desired = true; -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); kbase_pm_do_poweron(kbdev, false); @@ -190,7 +181,6 @@ void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev) if (kbdev->pm.backend.poweroff_wait_in_progress) return; -#if MALI_USE_CSF CSTD_UNUSED(shaders_desired); /* Invoke the MCU state machine to send a request to FW for updating * the mask of shader cores that can be used for allocation of @@ -198,18 +188,6 @@ void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev) */ if (kbase_pm_is_mcu_desired(kbdev)) kbase_pm_update_state(kbdev); -#else - /* In protected transition, don't allow outside shader core request - * affect transition, return directly - */ - if (kbdev->pm.backend.protected_transition_override) - return; - - shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); - - if (shaders_desired && kbase_pm_is_l2_desired(kbdev)) - kbase_pm_update_state(kbdev); -#endif } void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) @@ -223,16 +201,6 @@ void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) if (kbdev->pm.backend.poweroff_wait_in_progress) return; -#if !MALI_USE_CSF - if (kbdev->pm.backend.protected_transition_override) - /* We are trying to change in/out of protected mode - force all - * cores off so that the L2 powers down - */ - shaders_desired = false; - else - shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); -#endif - if (kbdev->pm.backend.shaders_desired != shaders_desired) { KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, kbdev->pm.backend.shaders_desired); @@ -274,7 +242,6 @@ const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) KBASE_EXPORT_TEST_API(kbase_pm_get_policy); -#if MALI_USE_CSF static int policy_change_wait_for_L2_off(struct kbase_device *kbdev) { long remaining; @@ -308,13 +275,11 @@ static int policy_change_wait_for_L2_off(struct kbase_device *kbdev) return err; } -#endif void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_policy *new_policy) { const struct kbase_pm_policy *old_policy; unsigned long flags; -#if MALI_USE_CSF unsigned int new_policy_csf_pm_sched_flags; bool sched_suspend; bool reset_gpu = false; @@ -322,14 +287,12 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic struct kbase_csf_scheduler *scheduler = NULL; u64 pwroff_ns; bool switching_to_always_on; -#endif KBASE_DEBUG_ASSERT(kbdev != NULL); KBASE_DEBUG_ASSERT(new_policy != NULL); KBASE_KTRACE_ADD(kbdev, PM_SET_POLICY, NULL, new_policy->id); -#if MALI_USE_CSF pwroff_ns = kbase_csf_firmware_get_mcu_core_pwroff_time(kbdev); switching_to_always_on = new_policy == &kbase_pm_always_on_policy_ops; if (pwroff_ns == 0 && !switching_to_always_on) { @@ -392,7 +355,6 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic if (sched_suspend) reset_gpu = policy_change_wait_for_L2_off(kbdev); -#endif kbase_pm_lock(kbdev); @@ -424,19 +386,19 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->pm.backend.pm_current_policy = new_policy; -#if MALI_USE_CSF kbdev->pm.backend.csf_pm_sched_flags = new_policy_csf_pm_sched_flags; /* New policy in place, release the clamping on mcu/L2 off state */ kbdev->pm.backend.policy_change_clamp_state_to_off = false; kbase_pm_update_state(kbdev); -#ifdef KBASE_PM_RUNTIME - if (kbase_pm_idle_groups_sched_suspendable(kbdev)) - clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); - else - set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); -#endif /* KBASE_PM_RUNTIME */ -#endif + if (IS_ENABLED(CONFIG_PM)) { + if (kbase_pm_idle_groups_sched_suspendable(kbdev)) + clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, + &kbdev->pm.backend.gpu_sleep_allowed); + else + set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* If any core power state changes were previously attempted, but @@ -452,7 +414,6 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic kbase_pm_context_idle_locked(kbdev); kbase_pm_unlock(kbdev); -#if MALI_USE_CSF /* Reverse the suspension done */ if (sched_suspend) kbase_csf_scheduler_pm_resume_no_lock(kbdev); @@ -469,7 +430,6 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic } mutex_unlock(&kbdev->pm.backend.policy_change_lock); -#endif } KBASE_EXPORT_TEST_API(kbase_pm_set_policy); diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_shader_states.h b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_shader_states.h index 8622ef78d363..84f1cbbcc95b 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_shader_states.h +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_pm_shader_states.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -42,11 +42,6 @@ * @WAIT_OFF_CORESTACK_ON: The shaders have been requested to power * off, but they remain on for the duration * of the hysteresis timer - * @WAIT_GPU_IDLE: The shaders partial poweroff needs to - * reach a state where jobs on the GPU are - * finished including jobs currently running - * and in the GPU queue because of - * GPU2017-861 * @WAIT_FINISHED_CORESTACK_ON: The hysteresis timer has expired * @L2_FLUSHING_CORESTACK_ON: The core stacks are on and the level 2 * cache is being flushed. @@ -67,9 +62,6 @@ KBASEP_SHADER_STATE(PEND_ON_CORESTACK_ON) KBASEP_SHADER_STATE(ON_CORESTACK_ON) KBASEP_SHADER_STATE(ON_CORESTACK_ON_RECHECK) KBASEP_SHADER_STATE(WAIT_OFF_CORESTACK_ON) -#if !MALI_USE_CSF -KBASEP_SHADER_STATE(WAIT_GPU_IDLE) -#endif /* !MALI_USE_CSF */ KBASEP_SHADER_STATE(WAIT_FINISHED_CORESTACK_ON) KBASEP_SHADER_STATE(L2_FLUSHING_CORESTACK_ON) KBASEP_SHADER_STATE(READY_OFF_CORESTACK_ON) diff --git a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_time.c index 0d00c2e7b5c0..5c7ec2b9c47d 100644 --- a/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_time.c +++ b/drivers/gpu/arm/valhall/backend/gpu/mali_kbase_time.c @@ -310,6 +310,9 @@ KBASE_EXPORT_TEST_API(kbase_get_timeout_ms); u64 kbase_backend_get_cycle_cnt(struct kbase_device *kbdev) { + if (kbase_io_is_aw_removed(kbdev)) + return 0; + return kbase_reg_read64_coherent(kbdev, GPU_CONTROL_ENUM(CYCLE_COUNT)); } @@ -330,6 +333,36 @@ u64 kbase_arch_timer_get_cntfrq(struct kbase_device *kbdev) return freq; } +static int kbase_gpu_timestamp_offset_read(void *data, u64 *val) +{ + struct kbase_device *kbdev = (struct kbase_device *)data; + + if (kbdev->backend_time.gpu_timestamp_offset == GPU_TIMESTAMP_OFFSET_INVALID) + return -EINVAL; + + *val = kbdev->backend_time.gpu_timestamp_offset; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(timestamp_offset_debugfs_fops, kbase_gpu_timestamp_offset_read, NULL, + "%lld\n"); + +void kbase_gpu_timestamp_offset_debugfs_init(struct kbase_device *kbdev) +{ + struct dentry *timestamp_offset_file; + + if (unlikely(!kbdev)) { + pr_warn("%s: kbdev is NULL\n", __func__); + return; + } + + timestamp_offset_file = debugfs_create_file("gpu_timestamp_offset", 0400, + kbdev->mali_debugfs_directory, kbdev, + ×tamp_offset_debugfs_fops); + if (IS_ERR_OR_NULL(timestamp_offset_file)) + dev_warn(kbdev->dev, "Failed to create gpu_timestamp_offset debugfs entry"); +} int kbase_backend_time_init(struct kbase_device *kbdev) { diff --git a/drivers/gpu/arm/valhall/build.bp b/drivers/gpu/arm/valhall/build.bp index 139632595381..5b72e1b17bc6 100644 --- a/drivers/gpu/arm/valhall/build.bp +++ b/drivers/gpu/arm/valhall/build.bp @@ -117,7 +117,6 @@ bob_defaults { "CONFIG_MALI_VALHALL_PLATFORM_NAME={{.mali_platform_name}}", "MALI_CUSTOMER_RELEASE={{.release}}", "MALI_UNIT_TEST={{.unit_test_code}}", - "MALI_USE_CSF={{.gpu_has_csf}}", "MALI_JIT_PRESSURE_LIMIT_BASE={{.jit_pressure_limit_base}}", // Start of CS experimental features definitions. @@ -133,7 +132,6 @@ bob_defaults { // is an umbrella feature that would be open for inappropriate use // (catch-all for experimental CS code without separating it into // different features). - "MALI_BASE_CSF_PERFORMANCE_TESTS={{.base_csf_performance_tests}}", ], } diff --git a/drivers/gpu/arm/valhall/context/Kbuild b/drivers/gpu/arm/valhall/context/Kbuild index 057693c40a30..cfc152edd81a 100644 --- a/drivers/gpu/arm/valhall/context/Kbuild +++ b/drivers/gpu/arm/valhall/context/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2012-2013, 2016-2017, 2020-2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012-2013, 2016-2017, 2020-2021, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -18,10 +18,6 @@ # # -valhall_kbase-y += context/mali_kbase_context.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += context/backend/mali_kbase_context_csf.o -else - valhall_kbase-y += context/backend/mali_kbase_context_jm.o -endif +valhall_kbase-y += \ + context/mali_kbase_context.o \ + context/backend/mali_kbase_context_csf.o diff --git a/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_csf.c b/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_csf.c index fe1dbfaca872..44cf8aba9b8f 100644 --- a/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_csf.c +++ b/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_csf.c @@ -91,6 +91,8 @@ static const struct kbase_context_init context_init[] = { "Common context initialization failed" }, { kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, "Memory pool group initialization failed" }, + { kbase_context_pgd_mem_pool_init, kbase_context_pgd_mem_pool_term, + "pgd memory pool initialization failed" }, { kbase_mem_evictable_init, kbase_mem_evictable_deinit, "Memory evictable initialization failed" }, { kbase_ctx_sched_init_ctx, NULL, NULL }, @@ -207,6 +209,7 @@ void kbase_destroy_context(struct kbase_context *kctx) wait_event(kbdev->pm.resume_wait, !kbase_pm_is_resuming(kbdev)); kbase_mem_pool_group_mark_dying(&kctx->mem_pools); + kbase_mem_pool_mark_dying(&kctx->pgd_mem_pool); kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); diff --git a/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_jm.c b/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_jm.c deleted file mode 100644 index ef474f625f63..000000000000 --- a/drivers/gpu/arm/valhall/context/backend/mali_kbase_context_jm.c +++ /dev/null @@ -1,266 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Base kernel context APIs for Job Manager GPUs - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if IS_ENABLED(CONFIG_DEBUG_FS) -#include -#include -#include -#include - -void kbase_context_debugfs_init(struct kbase_context *const kctx) -{ - kbase_debug_mem_view_init(kctx); - kbase_debug_mem_zones_init(kctx); - kbase_debug_mem_allocs_init(kctx); - kbase_mem_pool_debugfs_init(kctx->kctx_dentry, kctx); - kbase_jit_debugfs_init(kctx); - kbasep_jd_debugfs_ctx_init(kctx); -} -KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); - -void kbase_context_debugfs_term(struct kbase_context *const kctx) -{ - debugfs_remove_recursive(kctx->kctx_dentry); -} -KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); -#else -void kbase_context_debugfs_init(struct kbase_context *const kctx) -{ - CSTD_UNUSED(kctx); -} -KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); - -void kbase_context_debugfs_term(struct kbase_context *const kctx) -{ - CSTD_UNUSED(kctx); -} -KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); -#endif /* CONFIG_DEBUG_FS */ - -static int kbase_context_kbase_kinstr_jm_init(struct kbase_context *kctx) -{ - return kbase_kinstr_jm_init(&kctx->kinstr_jm); -} - -static void kbase_context_kbase_kinstr_jm_term(struct kbase_context *kctx) -{ - kbase_kinstr_jm_term(kctx->kinstr_jm); -} - -static int kbase_context_kbase_timer_setup(struct kbase_context *kctx) -{ - kbase_timer_setup(&kctx->soft_job_timeout, kbasep_soft_job_timeout_worker); - - return 0; -} - -static int kbase_context_submit_check(struct kbase_context *kctx) -{ - struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; - unsigned long irq_flags = 0; - - base_context_create_flags const flags = kctx->create_flags; - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); - - /* Translate the flags */ - if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) - kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); - - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return 0; -} - -static void kbase_context_flush_jobs(struct kbase_context *kctx) -{ - kbase_jd_zap_context(kctx); - flush_workqueue(kctx->jctx.job_done_wq); -} - -/** - * kbase_context_free - Free kcontext at its destruction - * - * @kctx: kcontext to be freed - */ -static void kbase_context_free(struct kbase_context *kctx) -{ - kbase_timeline_post_kbase_context_destroy(kctx); - - vfree(kctx); -} - -static const struct kbase_context_init context_init[] = { - { NULL, kbase_context_free, NULL }, - { kbase_context_common_init, kbase_context_common_term, - "Common context initialization failed" }, - { kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, - "Memory pool group initialization failed" }, - { kbase_mem_evictable_init, kbase_mem_evictable_deinit, - "Memory evictable initialization failed" }, - { kbase_ctx_sched_init_ctx, NULL, NULL }, - { kbase_context_mmu_init, kbase_context_mmu_term, "MMU initialization failed" }, - { kbase_context_mem_alloc_page, kbase_context_mem_pool_free, "Memory alloc page failed" }, - { kbase_region_tracker_init, kbase_region_tracker_term, - "Region tracker initialization failed" }, - { kbase_sticky_resource_init, kbase_context_sticky_resource_term, - "Sticky resource initialization failed" }, - { kbase_jit_init, kbase_jit_term, "JIT initialization failed" }, - { kbase_context_kbase_kinstr_jm_init, kbase_context_kbase_kinstr_jm_term, - "JM instrumentation initialization failed" }, - { kbase_context_kbase_timer_setup, NULL, "Timers initialization failed" }, - { kbase_event_init, kbase_event_cleanup, "Event initialization failed" }, - { kbasep_js_kctx_init, kbasep_js_kctx_term, "JS kctx initialization failed" }, - { kbase_jd_init, kbase_jd_exit, "JD initialization failed" }, - { kbase_context_submit_check, NULL, "Enabling job submission failed" }, -#if IS_ENABLED(CONFIG_DEBUG_FS) - { kbase_debug_job_fault_context_init, kbase_debug_job_fault_context_term, - "Job fault context initialization failed" }, -#endif - { kbasep_platform_context_init, kbasep_platform_context_term, - "Platform callback for kctx initialization failed" }, - { NULL, kbase_context_flush_jobs, NULL }, - { kbase_context_add_to_dev_list, kbase_context_remove_from_dev_list, - "Adding kctx to device failed" }, -}; - -static void kbase_context_term_partial(struct kbase_context *kctx, unsigned int i) -{ - while (i-- > 0) { - if (context_init[i].term) - context_init[i].term(kctx); - } -} - -struct kbase_context *kbase_create_context(struct kbase_device *kbdev, bool is_compat, - base_context_create_flags const flags, - unsigned long const api_version, struct file *const filp) -{ - struct kbase_context *kctx; - unsigned int i = 0; - - if (WARN_ON(!kbdev)) - return NULL; - - /* Validate flags */ - if (WARN_ON(flags != (flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS))) - return NULL; - - /* zero-inited as lot of code assume it's zero'ed out on create */ - kctx = vzalloc(sizeof(*kctx)); - if (WARN_ON(!kctx)) - return NULL; - - kctx->kbdev = kbdev; - kctx->api_version = api_version; - kctx->filp = filp; - kctx->create_flags = flags; - - if (is_compat) - kbase_ctx_flag_set(kctx, KCTX_COMPAT); -#if defined(CONFIG_64BIT) - else - kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); -#endif /* defined(CONFIG_64BIT) */ - - for (i = 0; i < ARRAY_SIZE(context_init); i++) { - int err = 0; - - if (context_init[i].init) - err = context_init[i].init(kctx); - - if (err) { - dev_err(kbdev->dev, "%s error = %d\n", context_init[i].err_mes, err); - - /* kctx should be freed by kbase_context_free(). - * Otherwise it will result in memory leak. - */ - WARN_ON(i == 0); - - kbase_context_term_partial(kctx, i); - return NULL; - } - } - - return kctx; -} -KBASE_EXPORT_SYMBOL(kbase_create_context); - -void kbase_destroy_context(struct kbase_context *kctx) -{ - struct kbase_device *kbdev; - - if (WARN_ON(!kctx)) - return; - - kbdev = kctx->kbdev; - if (WARN_ON(!kbdev)) - return; - - /* Context termination could happen whilst the system suspend of - * the GPU device is ongoing or has completed. It has been seen on - * Customer side that a hang could occur if context termination is - * not blocked until the resume of GPU device. - */ - if (kbase_has_arbiter(kbdev)) - atomic_inc(&kbdev->pm.gpu_users_waiting); - while (kbase_pm_context_active_handle_suspend(kbdev, - KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { - dev_dbg(kbdev->dev, "Suspend in progress when destroying context"); - wait_event(kbdev->pm.resume_wait, !kbase_pm_is_suspending(kbdev)); - } - - /* Have synchronized against the System suspend and incremented the - * pm.active_count. So any subsequent invocation of System suspend - * callback would get blocked. - * If System suspend callback was already in progress then the above loop - * would have waited till the System resume callback has begun. - * So wait for the System resume callback to also complete as we want to - * avoid context termination during System resume also. - */ - wait_event(kbdev->pm.resume_wait, !kbase_pm_is_resuming(kbdev)); - - if (kbase_has_arbiter(kbdev)) - atomic_dec(&kbdev->pm.gpu_users_waiting); - - kbase_mem_pool_group_mark_dying(&kctx->mem_pools); - - kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); - - kbase_pm_context_idle(kbdev); -} -KBASE_EXPORT_SYMBOL(kbase_destroy_context); diff --git a/drivers/gpu/arm/valhall/context/mali_kbase_context.c b/drivers/gpu/arm/valhall/context/mali_kbase_context.c index 5126f3f6d82a..34f40073779e 100644 --- a/drivers/gpu/arm/valhall/context/mali_kbase_context.c +++ b/drivers/gpu/arm/valhall/context/mali_kbase_context.c @@ -203,6 +203,7 @@ int kbase_context_common_init(struct kbase_context *kctx) } } + kctx->offslot_ts = 0; return err; } @@ -296,8 +297,11 @@ void kbase_context_common_term(struct kbase_context *kctx) int kbase_context_mem_pool_group_init(struct kbase_context *kctx) { - return kbase_mem_pool_group_init(&kctx->mem_pools, kctx->kbdev, - &kctx->kbdev->mem_pool_defaults, &kctx->kbdev->mem_pools); + size_t const small_max_size = KBASE_MEM_POOL_MAX_SIZE_KCTX; + size_t const large_max_size = small_max_size >> (KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER - + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER); + return kbase_mem_pool_group_init(&kctx->mem_pools, kctx->kbdev, small_max_size, + large_max_size, NULL); } void kbase_context_mem_pool_group_term(struct kbase_context *kctx) @@ -305,6 +309,20 @@ void kbase_context_mem_pool_group_term(struct kbase_context *kctx) kbase_mem_pool_group_term(&kctx->mem_pools); } +int kbase_context_pgd_mem_pool_init(struct kbase_context *kctx) +{ + int err = kbase_mem_pool_init_no_reclaim(&kctx->pgd_mem_pool, + BASE_PGD_MEM_POOL_MAX_SIZE_KBDEV, + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, 0, + kctx->kbdev); + return err; +} + +void kbase_context_pgd_mem_pool_term(struct kbase_context *kctx) +{ + kbase_mem_pool_term(&kctx->pgd_mem_pool); +} + int kbase_context_mmu_init(struct kbase_context *kctx) { return kbase_mmu_init(kctx->kbdev, &kctx->mmu, kctx, diff --git a/drivers/gpu/arm/valhall/context/mali_kbase_context_internal.h b/drivers/gpu/arm/valhall/context/mali_kbase_context_internal.h index 8d9b39419586..5552aa3e036e 100644 --- a/drivers/gpu/arm/valhall/context/mali_kbase_context_internal.h +++ b/drivers/gpu/arm/valhall/context/mali_kbase_context_internal.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -50,6 +50,9 @@ void kbase_context_common_term(struct kbase_context *kctx); int kbase_context_mem_pool_group_init(struct kbase_context *kctx); void kbase_context_mem_pool_group_term(struct kbase_context *kctx); +int kbase_context_pgd_mem_pool_init(struct kbase_context *kctx); +void kbase_context_pgd_mem_pool_term(struct kbase_context *kctx); + int kbase_context_mmu_init(struct kbase_context *kctx); void kbase_context_mmu_term(struct kbase_context *kctx); diff --git a/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.c b/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.c index ecf19001c345..257d64b609c1 100644 --- a/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.c +++ b/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.c @@ -897,7 +897,6 @@ void kbase_ipa_control_handle_gpu_reset_post(struct kbase_device *kbdev) } KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_reset_post); -#ifdef KBASE_PM_RUNTIME void kbase_ipa_control_handle_gpu_sleep_enter(struct kbase_device *kbdev) { lockdep_assert_held(&kbdev->hwaccess_lock); @@ -931,7 +930,6 @@ void kbase_ipa_control_handle_gpu_sleep_exit(struct kbase_device *kbdev) } } KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_sleep_exit); -#endif #if MALI_UNIT_TEST void kbase_ipa_control_rate_change_notify_test(struct kbase_device *kbdev, u32 clk_index, diff --git a/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.h b/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.h index 80aabc1022c5..0d1fe664d9a5 100644 --- a/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.h +++ b/drivers/gpu/arm/valhall/csf/ipa_control/mali_kbase_csf_ipa_control.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -197,7 +197,6 @@ void kbase_ipa_control_handle_gpu_reset_pre(struct kbase_device *kbdev); */ void kbase_ipa_control_handle_gpu_reset_post(struct kbase_device *kbdev); -#ifdef KBASE_PM_RUNTIME /** * kbase_ipa_control_handle_gpu_sleep_enter - Handle the pre GPU Sleep event * @@ -222,7 +221,6 @@ void kbase_ipa_control_handle_gpu_sleep_enter(struct kbase_device *kbdev); * was called previously. */ void kbase_ipa_control_handle_gpu_sleep_exit(struct kbase_device *kbdev); -#endif #if MALI_UNIT_TEST /** diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c index c0231abe8d5f..26ebe40690cc 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -211,6 +211,11 @@ static void kernel_free_user_io_pages(struct kbase_context *kctx, struct tagged_ kbase_gpu_vm_lock(kctx); vunmap(user_io_addr); + if (kctx->csf.user_io.vma != NULL) { + zap_vma_ptes(kctx->csf.user_io.vma, kctx->csf.user_io.vma->vm_start, + KBASEP_NUM_CS_USER_IO_PAGES * PAGE_SIZE); + kctx->csf.user_io.vma = NULL; + } WARN_ON(atomic_read(&kctx->permanent_mapped_pages) < KBASEP_NUM_CS_USER_IO_PAGES); atomic_sub(KBASEP_NUM_CS_USER_IO_PAGES, &kctx->permanent_mapped_pages); @@ -319,10 +324,9 @@ void kbase_csf_free_command_stream_user_pages(struct kbase_context *kctx, struct * This condition is only true when mali_kbase_supports_csg_cs_user_page_allocation() * is disable. */ - if (!mali_kbase_supports_csg_cs_user_page_allocation(kctx->api_version)) { + if (!mali_kbase_supports_csg_cs_user_page_allocation(kctx->api_version)) kernel_free_user_io_pages(kctx, queue->phys, queue->user_io_addr); - queue->user_io_addr = NULL; - } + queue->user_io_addr = NULL; /* The user_io_gpu_va should have been unmapped inside the scheduler */ WARN_ONCE(queue->user_io_gpu_va, "Userio pages appears still have mapping"); @@ -773,6 +777,12 @@ int kbase_csf_queue_bind(struct kbase_context *kctx, union kbase_ioctl_cs_queue_ if (bind->in.csi_index >= max_streams) goto out; + if (queue->user_io_addr != NULL) { + dev_err(kctx->kbdev->dev, "Queue with stale user_io address: %pK", + (void *)queue->user_io_addr); + goto out; + } + if (group->run_state == KBASE_CSF_GROUP_TERMINATED) goto out; @@ -1852,8 +1862,6 @@ void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, struct kbase_fault * void kbase_csf_ctx_term(struct kbase_context *kctx) { struct kbase_device *kbdev = kctx->kbdev; - struct kbase_as *as = NULL; - unsigned long flags; u32 i; int err; bool reset_prevented = false; @@ -1922,15 +1930,57 @@ void kbase_csf_ctx_term(struct kbase_context *kctx) flush_work(&kctx->kbdev->csf.glb_fatal_work); /* A work item to handle page_fault/bus_fault/gpu_fault could be - * pending for the outgoing context. Flush the workqueue that will - * execute that work item. + * pending for the outgoing context which we no longer care about. + * Ensure that the context won't be accessed anymore by the fault + * workers. */ - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); - if (kctx->as_nr != KBASEP_AS_NR_INVALID) - as = &kctx->kbdev->as[kctx->as_nr]; - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); - if (as) - flush_workqueue(as->pf_wq); + while (true) { + unsigned long flags; + int refcount; + + mutex_lock(&kbdev->mmu_hw_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + refcount = atomic_read(&kctx->refcount); + if ((refcount != 0) && !WARN_ON_ONCE(kctx->as_nr == KBASEP_AS_NR_INVALID)) { + struct kbase_as *as = &kctx->kbdev->as[kctx->as_nr]; + int new_refcount; + + dev_dbg(kbdev->dev, + "Waiting for pending fault worker to complete when terminating context (%d_%d)", + kctx->tgid, kctx->id); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + flush_workqueue(as->pf_wq); + + new_refcount = atomic_read(&kctx->refcount); + if (refcount != new_refcount) { + /* Fault workers executed and released some references, re-check */ + continue; + } else { + /* Waiting for pending fault workers to execute was not effective, + * we're going to forcefully de-assign the AS from this context + * because nothing else should still be accessing the context at + * this point. + * + * This should never happen and a WARN_ON() would be printed by + * kbase_ctx_sched_remove_ctx() if the refcount is non-zero. + */ + dev_warn( + kbdev->dev, + "No fault workers executed, %d refs remain for terminating context (%d_%d)", + new_refcount, kctx->tgid, kctx->id); + kbase_ctx_sched_remove_ctx(kctx); + } + } else { + kbase_ctx_sched_remove_ctx_nolock(kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); + } + + break; + } mutex_lock(&kctx->csf.lock); @@ -2439,27 +2489,33 @@ static void handle_progress_timer_events(struct kbase_device *const kbdev, unsig u32 max_csg_slots = kbdev->csf.global_iface.group_num; u32 csg_nr; struct kbase_queue_group *group = NULL; + bool aw_removed; kbase_csf_scheduler_spin_lock_assert_held(kbdev); if (likely(bitmap_empty(slot_mask, BASEP_QUEUE_GROUP_MAX))) return; - /* Log each timeout and Update timestamp of compute progress timeout */ - for_each_set_bit(csg_nr, slot_mask, max_csg_slots) { - group = kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; - group->progress_timer_state = kbase_csf_fw_io_group_read(&kbdev->csf.fw_io, csg_nr, - CSG_PROGRESS_TIMER_STATE); + aw_removed = kbase_io_is_aw_removed(kbdev); - dev_info( - kbdev->dev, - "[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u with state %x", - kbase_backend_get_cycle_cnt(kbdev), group->handle, group->kctx->tgid, - group->kctx->id, csg_nr, group->progress_timer_state); + if (!aw_removed) + /* Log each timeout and Update timestamp of compute progress timeout */ + for_each_set_bit(csg_nr, slot_mask, max_csg_slots) { + group = kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; + group->progress_timer_state = kbase_csf_fw_io_group_read( + &kbdev->csf.fw_io, csg_nr, CSG_PROGRESS_TIMER_STATE); - if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == - CSG_PROGRESS_TIMER_STATE_COMPUTE) - kbdev->csf.compute_progress_timeout_cc = kbase_backend_get_cycle_cnt(kbdev); - } + dev_info( + kbdev->dev, + "[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u with state %x", + kbase_backend_get_cycle_cnt(kbdev), group->handle, + group->kctx->tgid, group->kctx->id, csg_nr, + group->progress_timer_state); + + if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == + CSG_PROGRESS_TIMER_STATE_COMPUTE) + kbdev->csf.compute_progress_timeout_cc = + kbase_backend_get_cycle_cnt(kbdev); + } /* Ignore fragment timeout if it is following a compute timeout. * Otherwise, terminate the command stream group. @@ -2470,8 +2526,8 @@ static void handle_progress_timer_events(struct kbase_device *const kbdev, unsig /* Check if it is a fragment timeout right after another compute timeout. * In such case, kill compute CSG and give fragment CSG a second chance */ - if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == - CSG_PROGRESS_TIMER_STATE_FRAGMENT) { + if (!aw_removed && CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == + CSG_PROGRESS_TIMER_STATE_FRAGMENT) { u64 cycle_counter = kbase_backend_get_cycle_cnt(kbdev); u64 compute_progress_timeout_cc = kbdev->csf.compute_progress_timeout_cc; @@ -3005,7 +3061,27 @@ static int process_cs_interrupts(struct kbase_queue_group *const group, u32 grou kbase_csf_fw_io_close(fw_io, fw_io_flags); } - /* PROTM_PEND and TILER_OOM can be safely ignored + /* PROTM_PEND request should be handled if the group + * is assigned a CSG slot in the future. We set + * protm_pending_bitmap here in case we skip saving + * the CS slots' state due to the group becoming active + * shortly after the suspension request is made. + */ + if ((cs_req & CS_REQ_PROTM_PEND_MASK) ^ (cs_ack & CS_ACK_PROTM_PEND_MASK)) { + KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_INTERRUPT_PROTM_PEND, group, + queue, cs_req ^ cs_ack); + + dev_dbg(kbdev->dev, + "Protected mode entry request for queue on csi %d bound to group-%d on slot %d", + queue->csi_index, group->handle, group->csg_nr); + + bitmap_set(group->protm_pending_bitmap, i, 1); + KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_PROTM_PEND_SET, group, queue, + group->protm_pending_bitmap[0]); + protm_pend = true; + } + + /* TILER_OOM can be safely ignored * because they will be raised again if the group * is assigned a CSG slot in future. */ @@ -3026,20 +3102,6 @@ static int process_cs_interrupts(struct kbase_queue_group *const group, u32 grou kbase_csf_handle_pending_oom_interrupt(queue, group_id); } - - if ((cs_req & CS_REQ_PROTM_PEND_MASK) ^ (cs_ack & CS_ACK_PROTM_PEND_MASK)) { - KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_INTERRUPT_PROTM_PEND, group, - queue, cs_req ^ cs_ack); - - dev_dbg(kbdev->dev, - "Protected mode entry request for queue on csi %d bound to group-%d on slot %d", - queue->csi_index, group->handle, group->csg_nr); - - bitmap_set(group->protm_pending_bitmap, i, 1); - KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_PROTM_PEND_SET, group, queue, - group->protm_pending_bitmap[0]); - protm_pend = true; - } } } @@ -3757,7 +3819,7 @@ void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev) * module unload path, so the page can be left uncleared before returning it * back to kbdev memory pool. */ - kbase_mem_pool_free(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, false); + kbase_mem_pool_free(&kbdev->fw_mem_pools.small, page, false); fput(kbdev->csf.db_filp); } @@ -3773,8 +3835,7 @@ int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev) if (IS_ERR(filp)) return PTR_ERR(filp); - ret = kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], 1, &phys, - false, NULL); + ret = kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, 1, &phys, false, NULL); if (ret <= 0) { fput(filp); @@ -3821,7 +3882,7 @@ void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev) * path, so the page can be left uncleared before returning it back to kbdev * memory pool. */ - kbase_mem_pool_free(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, false); + kbase_mem_pool_free(&kbdev->fw_mem_pools.small, page, false); fput(kbdev->csf.user_reg.filp); } } @@ -3841,8 +3902,7 @@ int kbase_csf_setup_dummy_user_reg_page(struct kbase_device *kbdev) return PTR_ERR(filp); } - if (kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], 1, &phys, - false, NULL) <= 0) { + if (kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, 1, &phys, false, NULL) <= 0) { fput(filp); return -ENOMEM; } diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_csg_debugfs.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_csg_debugfs.c index 3d614514e8bd..7a53bb7ad556 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_csg_debugfs.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_csg_debugfs.c @@ -50,10 +50,8 @@ static const char *scheduler_state_to_string(struct kbase_device *kbdev, return "INACTIVE"; case SCHED_SUSPENDED: return "SUSPENDED"; -#ifdef KBASE_PM_RUNTIME case SCHED_SLEEPING: return "SLEEPING"; -#endif default: dev_warn(kbdev->dev, "Unknown Scheduler state %d", sched_state); return NULL; @@ -249,10 +247,8 @@ static ssize_t kbase_csf_debugfs_scheduler_state_set(struct file *file, const ch if (sysfs_streq(buf, "SUSPENDED")) kbase_csf_scheduler_pm_suspend(kbdev); -#ifdef KBASE_PM_RUNTIME - else if (sysfs_streq(buf, "SLEEPING")) + else if (IS_ENABLED(CONFIG_PM) && sysfs_streq(buf, "SLEEPING")) kbase_csf_scheduler_force_sleep(kbdev); -#endif else if (sysfs_streq(buf, "INACTIVE")) kbase_csf_scheduler_force_wakeup(kbdev); else { diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h index 9f62587ba2cc..16741079cb36 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h @@ -927,6 +927,17 @@ struct kbase_csf_event { spinlock_t lock; }; +/** + * struct kbase_csf_user_io_context - Object containing members to manage the mapping + * of USER io page for a context. + * + * @vma: Pointer to the VMA corresponding to the virtual mapping + * of the USER io page. + */ +struct kbase_csf_user_io_context { + struct vm_area_struct *vma; +}; + /** * struct kbase_csf_user_reg_context - Object containing members to manage the mapping * of USER Register page for a context. @@ -986,6 +997,7 @@ struct kbase_csf_user_reg_context { * @cpu_queue: CPU queue information. Only be available when DEBUG_FS * is enabled. * @user_reg: Collective information to support mapping to USER Register page. + * @user_io: Collective information to support mapping to USER IO page. * @pending_sync_update: Indicates that kbase_csf_scheduler_kthread() should * handle SYNC_UPDATE event for this context. This would * be set to false when the work is done. This is used @@ -1006,6 +1018,7 @@ struct kbase_csf_context { struct kbase_csf_scheduler_context sched; struct kbase_csf_cpu_queue_context cpu_queue; struct kbase_csf_user_reg_context user_reg; + struct kbase_csf_user_io_context user_io; atomic_t pending_sync_update; }; @@ -1043,6 +1056,17 @@ struct kbase_csf_csg_slot { u8 priority; }; +/** + * struct kbase_csf_heap_reclaim_offslot - Setting for reclaiming offslot CSG's heap. + * + * @timeout_ms: Reclaim from CSGs being offslot longer than this. + * @pages: Max number of pages for each reclaim. + */ +struct kbase_csf_heap_reclaim_offslot { + u32 timeout_ms; + u32 pages; +}; + /** * struct kbase_csf_sched_heap_reclaim_mgr - Object for managing tiler heap reclaim * kctx lists inside the CSF device's scheduler. @@ -1052,12 +1076,14 @@ struct kbase_csf_csg_slot { * lists track the kctxs attached to the reclaim manager. * @unused_pages: Estimated number of unused pages from the @ctxlist array. The * number is indicative for use with reclaim shrinker's count method. + * @offslot_setting: Setting for reclaiming offslot CSG's heap. */ struct kbase_csf_sched_heap_reclaim_mgr { DEFINE_KBASE_SHRINKER heap_reclaim; struct list_head ctx_lists[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; atomic_t unused_pages; + struct kbase_csf_heap_reclaim_offslot offslot_setting; }; /** @@ -1177,6 +1203,9 @@ struct kbase_csf_mcu_shared_regions { * perform a scheduling tock. * @pending_gpu_idle_work: Indicates that kbase_csf_scheduler_kthread() should * handle the GPU IDLE event. + * @pending_runtime_suspend_work: Indicates that kbase_csf_scheduler_kthread() + * should proceed to suspend the GPU as part of + * handling the runtime suspend event. * @pending_power_off_work: Indicates that kbase_csf_scheduler_kthread() should * proceed to power off the GPU. * @ping_work: Work item that would ping the firmware at regular @@ -1228,10 +1257,13 @@ struct kbase_csf_mcu_shared_regions { * @mcu_regs_data: Scheduler MCU shared regions data for managing the * shared interface mappings for on-slot queues and * CSG suspend buffers. - * @kthread_signal: Used to wake up the GPU queue submission - * thread when a queue needs attention. - * @kthread_running: Whether the GPU queue submission thread should keep - * executing. + * @kthread_signal: Used to wake up the main CSF scheduler thread + * to handle pending work items. + * @kthread_running: Set to true to indicate that the CSF scheduler + * thread will handle work items. Work items that + * are handled by this thread all require the schduler + * mutex lock, thus are serialised and executed in a + * predefined order. * @gpuq_kthread: Dedicated thread primarily used to handle * latency-sensitive tasks such as GPU queue * submissions. @@ -1272,6 +1304,7 @@ struct kbase_csf_scheduler { atomic_t pending_tick_work; atomic_t pending_tock_work; atomic_t pending_gpu_idle_work; + atomic_t pending_runtime_suspend_work; atomic_t pending_power_off_work; struct delayed_work ping_work; struct kbase_context *top_kctx; @@ -1813,7 +1846,8 @@ struct kbase_csf_user_reg { * @fw_io: Firmware I/O interface. * @compute_progress_timeout_cc: Value of GPU cycle count register when progress * timer timeout is reported for the compute iterator. - * @num_doorbells: Number of doorbells supported by the GPU. + * @neural_allowed_mask: A mask for optionally disabling neural cores across all CSGs + * @num_doorbells: Number of doorbells supported by the GPU. */ struct kbase_csf_device { struct kbase_mmu_table mcu_mmu; @@ -1877,6 +1911,7 @@ struct kbase_csf_device { u32 page_fault_cnt; struct kbase_csf_fw_io fw_io; u64 compute_progress_timeout_cc; + u64 neural_allowed_mask; u32 num_doorbells; }; diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_event.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_event.c index 08faace8ef06..a91d99816a9c 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_event.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_event.c @@ -94,11 +94,11 @@ static void sync_update_notify_gpu(struct kbase_context *kctx) spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); can_notify_gpu = kbase_io_is_gpu_powered(kctx->kbdev); -#ifdef KBASE_PM_RUNTIME - if (kctx->kbdev->pm.backend.db_mirror_interrupt_enabled || - kctx->kbdev->pm.backend.gpu_sleep_mode_active) - can_notify_gpu = false; -#endif + if (IS_ENABLED(CONFIG_PM)) { + if (kctx->kbdev->pm.backend.db_mirror_interrupt_enabled || + kctx->kbdev->pm.backend.gpu_sleep_mode_active) + can_notify_gpu = false; + } if (can_notify_gpu) { diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.c index d1d53a075cbb..5680d2c21116 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -377,6 +377,15 @@ void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev) kbase_csf_firmware_disable_mcu_wait(kbdev); } +static void kbasep_hwcnt_init_on_boot(struct kbase_device *kbdev) +{ + kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(&kbdev->hwcnt_gpu_iface); + kbase_hwcnt_backend_csf_set_hw_availability(&kbdev->hwcnt_gpu_iface, + kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); +} + static void wait_for_firmware_boot(struct kbase_device *kbdev) { long wait_timeout; @@ -400,6 +409,8 @@ static void wait_for_firmware_boot(struct kbase_device *kbdev) wait_timeout); if (!remaining) dev_err(kbdev->dev, "Timed out waiting for fw boot completion"); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); + kbdev->csf.interrupt_received = false; } @@ -414,6 +425,8 @@ static void enable_mcu(struct kbase_device *kbdev) static void boot_csf_firmware(struct kbase_device *kbdev) { + kbasep_hwcnt_init_on_boot(kbdev); + enable_mcu(kbdev); wait_for_firmware_boot(kbdev); @@ -440,6 +453,12 @@ static int wait_ready(struct kbase_device *kbdev) if (!err) return 0; + /* Bailout on GPU_LOST without RESET */ + if (err == -ENODEV) { + dev_warn(kbdev->dev, "%s: AW removed, bailing out", __func__); + return 0; + } + dev_err(kbdev->dev, "AS_ACTIVE bit stuck for MCU AS. Might be caused by unstable GPU clk/pwr or faulty system"); @@ -835,10 +854,10 @@ retry_alloc: } } else { if (!reuse_pages) { - ret = kbase_mem_pool_alloc_pages( - kbase_mem_pool_group_select(kbdev, KBASE_MEM_GROUP_CSF_FW, - is_small_page), - num_pages_aligned, phys, false, NULL); + ret = kbase_mem_pool_alloc_pages(is_small_page ? + &kbdev->fw_mem_pools.small : + &kbdev->fw_mem_pools.large, + num_pages_aligned, phys, false, NULL); } } @@ -971,10 +990,9 @@ out: kbase_csf_protected_memory_free(kbdev, pma, num_pages_aligned, is_small_page); } else { - kbase_mem_pool_free_pages( - kbase_mem_pool_group_select(kbdev, KBASE_MEM_GROUP_CSF_FW, - is_small_page), - num_pages_aligned, phys, false, false); + kbase_mem_pool_free_pages(is_small_page ? &kbdev->fw_mem_pools.small : + &kbdev->fw_mem_pools.large, + num_pages_aligned, phys, false, false); } kfree(phys); } @@ -1386,6 +1404,11 @@ static int parse_capabilities(struct kbase_device *kbdev) else iface->instr_features = 0; + if (iface->version >= kbase_csf_interface_version(1, 1, 0)) + iface->prfcnt_features = shared_info[GLB_PRFCNT_FEATURES / 4]; + else + iface->prfcnt_features = 0; + if ((GROUP_CONTROL_0 + (unsigned long)iface->group_num * iface->group_stride) > (interface->num_pages * PAGE_SIZE)) { dev_err(kbdev->dev, @@ -1600,9 +1623,7 @@ static void set_global_request(struct kbase_csf_fw_io *fw_io, u32 const req_mask kbase_csf_fw_io_global_write_mask(fw_io, GLB_REQ, glb_req, req_mask); } -static void enable_endpoints_global(struct kbase_csf_fw_io *fw_io, - - u64 const shader_core_mask) +static void enable_endpoints_global(struct kbase_csf_fw_io *fw_io, u64 const shader_core_mask) { kbase_csf_fw_io_assert_opened(fw_io); @@ -1905,6 +1926,11 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) kbasep_enable_rtu(kbdev); /* Update shader core allocation enable mask */ + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + if (kbase_io_is_gpu_powered(kbdev)) + kbase_reg_write64(kbdev, GPU_GOVERNOR_ENUM(GOV_CORE_MASK), + kbase_pm_ca_get_gov_core_mask(kbdev)); + enable_endpoints_global(fw_io, core_mask); set_shader_poweroff_timer(fw_io); @@ -1935,6 +1961,13 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) if (kbdev->pm.backend.has_host_pwr_iface) ack_irq_mask |= GLB_ACK_IRQ_MASK_STATE_MASK; + + /* Do not modify PRFCNT bits of GLB_ACK_IRQ_MASK. */ + ack_irq_mask |= + (kbase_csf_fw_io_global_input_read(&kbdev->csf.fw_io, GLB_ACK_IRQ_MASK) & + (GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK | GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK | + GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK | GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK)); + /* Unmask the interrupts */ kbase_csf_fw_io_global_write(fw_io, GLB_ACK_IRQ_MASK, ack_irq_mask); @@ -1965,14 +1998,32 @@ exit: */ static int global_init_on_boot(struct kbase_device *const kbdev) { - unsigned long flags; + unsigned long flags, scheduler_lock_flags; u64 core_mask; int ret = 0; u32 request_mask = CSF_GLB_REQ_CFG_MASK; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + core_mask = kbase_pm_ca_get_core_mask(kbdev); + kbdev->csf.firmware_hctl_core_pwr = kbase_pm_no_mcu_core_pwroff(kbdev); + + /* Enable the HWC context for the case of MCU controlling the shader core power. + * HWC context will be left enabled throughout the MCU power cycles, unless + * there is a GPU reset. + * + * In case of GPU reset, HWC needs to be disabled to ensure + * the correct HWC backend functionality. The context then gets reenabled during + * the reboot. + */ + if (!kbdev->csf.firmware_hctl_core_pwr) { + kbase_csf_scheduler_spin_lock(kbdev, &scheduler_lock_flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, scheduler_lock_flags); + kbdev->pm.backend.hwcnt_disabled = false; + kbdev->pm.backend.hwcnt_desired = true; + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); global_init(kbdev, core_mask); @@ -1996,10 +2047,25 @@ static int global_init_on_boot(struct kbase_device *const kbdev) void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev, u64 core_mask) { + unsigned long flags; lockdep_assert_held(&kbdev->hwaccess_lock); kbdev->csf.glb_init_request_pending = true; kbdev->csf.firmware_hctl_core_pwr = kbase_pm_no_mcu_core_pwroff(kbdev); + if (!kbdev->csf.firmware_hctl_core_pwr) { + /* The HWC context reenabling is needed if: + * - shader core power control was previously under host. + * - context was disabled when preparing for a GPU reset. + */ + kbdev->pm.backend.hwcnt_desired = true; + if (kbdev->pm.backend.hwcnt_disabled) { + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + kbdev->pm.backend.hwcnt_disabled = false; + } + } + global_init(kbdev, core_mask); } @@ -2020,9 +2086,6 @@ void kbase_csf_firmware_update_core_attr(struct kbase_device *kbdev, bool update struct kbase_csf_fw_io *fw_io = &kbdev->csf.fw_io; unsigned long flags, fw_io_flags; - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) - core_mask = U64_MAX; - lockdep_assert_held(&kbdev->hwaccess_lock); kbase_csf_scheduler_spin_lock(kbdev, &flags); @@ -2083,6 +2146,7 @@ static void kbase_csf_firmware_reload_worker(struct work_struct *work) return; /* Reboot the firmware */ + kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(&kbdev->hwcnt_gpu_iface); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_csf_firmware_enable_mcu(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -2096,9 +2160,13 @@ void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) if (kbdev->csf.firmware_reload_needed) { kbdev->csf.firmware_reload_needed = false; + /* MCU cold boot requested. */ queue_work(system_wq, &kbdev->csf.firmware_reload_work); } else { - kbase_csf_firmware_enable_mcu(kbdev); + /* MCU shall not boot while reset is in progress */ + if (likely(!kbdev->pm.backend.in_reset)) + /* MCU warm boot requested. */ + kbase_csf_firmware_enable_mcu(kbdev); } } @@ -2129,6 +2197,11 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) dev_err(kbdev->dev, "Version check failed in firmware reboot."); KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_REBOOT, NULL, 0u); + kbase_hwcnt_backend_csf_set_hw_availability(&kbdev->hwcnt_gpu_iface, + kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); /* Tell MCU state machine to transit to next state */ kbdev->csf.firmware_reloaded = true; @@ -2419,10 +2492,9 @@ int kbase_csf_firmware_late_init(struct kbase_device *kbdev) kbdev->csf.gpu_idle_hysteresis_ns = FIRMWARE_IDLE_HYSTERESIS_TIME_NS; -#ifdef KBASE_PM_RUNTIME - if (kbase_pm_gpu_sleep_allowed(kbdev)) + if (IS_ENABLED(CONFIG_PM) && kbase_pm_gpu_sleep_allowed(kbdev)) kbdev->csf.gpu_idle_hysteresis_ns /= FIRMWARE_IDLE_HYSTERESIS_GPU_SLEEP_SCALER; -#endif + WARN_ON(!kbdev->csf.gpu_idle_hysteresis_ns); kbdev->csf.gpu_idle_dur_count = convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &no_modifier); @@ -2751,10 +2823,11 @@ void kbase_csf_firmware_unload_term(struct kbase_device *kbdev) interface->num_pages_aligned, interface->is_small_page); } else { - kbase_mem_pool_free_pages( - kbase_mem_pool_group_select(kbdev, KBASE_MEM_GROUP_CSF_FW, - interface->is_small_page), - interface->num_pages_aligned, interface->phys, true, false); + kbase_mem_pool_free_pages(interface->is_small_page ? + &kbdev->fw_mem_pools.small : + &kbdev->fw_mem_pools.large, + interface->num_pages_aligned, + interface->phys, true, false); } kfree(interface->phys); @@ -3128,7 +3201,6 @@ void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev) enable_mcu(kbdev); } -#ifdef KBASE_PM_RUNTIME void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev) { struct kbase_csf_fw_io *fw_io = &kbdev->csf.fw_io; @@ -3169,7 +3241,6 @@ bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev) } KBASE_EXPORT_TEST_API(kbase_csf_firmware_is_mcu_in_sleep); -#endif /* KBASE_PM_RUNTIME */ bool kbase_csf_firmware_mcu_halt_req_complete(struct kbase_device *kbdev) { @@ -3359,8 +3430,7 @@ int kbase_csf_firmware_mcu_shared_mapping_init(struct kbase_device *kbdev, unsig if (!page_list) goto page_list_alloc_error; - ret = kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, - phys, false, NULL); + ret = kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, NULL); if (ret <= 0) goto phys_mem_pool_alloc_error; @@ -3408,8 +3478,7 @@ va_region_add_error: va_region_alloc_error: vunmap(cpu_addr); vmap_error: - kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, phys, - false, false); + kbase_mem_pool_free_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, false); phys_mem_pool_alloc_error: kfree(page_list); @@ -3441,16 +3510,14 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev, /* This is on module unload path, so the pages can be left uncleared before * returning them back to kbdev memory pool. */ - kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], - csf_mapping->num_pages, csf_mapping->phys, false, false); + kbase_mem_pool_free_pages(&kbdev->fw_mem_pools.small, csf_mapping->num_pages, + csf_mapping->phys, false, false); } vunmap(csf_mapping->cpu_addr); kfree(csf_mapping->phys); } -#ifdef KBASE_PM_RUNTIME - void kbase_csf_firmware_soi_update(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; @@ -3592,5 +3659,3 @@ int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbd return 0; } - -#endif /* KBASE_PM_RUNTIME */ diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.h index e2c627902097..6762bd22ec3e 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware.h @@ -161,6 +161,7 @@ struct kbase_csf_cmd_stream_group_info { * CSG capability structures. * @prfcnt_size: Performance counters size. * @instr_features: Instrumentation features. (csf >= 1.1.0) + * @prfcnt_features: Performance Counter features. * @groups: Address of an array of CSG capability structures. */ struct kbase_csf_global_iface { @@ -171,6 +172,7 @@ struct kbase_csf_global_iface { u32 group_stride; u32 prfcnt_size; u32 instr_features; + u32 prfcnt_features; struct kbase_csf_cmd_stream_group_info *groups; }; @@ -477,7 +479,6 @@ void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev); */ void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev); -#ifdef KBASE_PM_RUNTIME /** * kbase_csf_firmware_trigger_mcu_sleep - Send the command to put MCU in sleep * state. @@ -496,7 +497,6 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev); */ bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev); -#endif /** * kbase_csf_firmware_trigger_reload() - Trigger the reboot of MCU firmware, for @@ -812,8 +812,6 @@ int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev); */ int kbase_csf_firmware_req_core_dump(struct kbase_device *const kbdev); -#ifdef KBASE_PM_RUNTIME - /** * kbase_csf_firmware_soi_update - Update FW Sleep-on-Idle config * @@ -842,7 +840,5 @@ void kbase_csf_firmware_glb_idle_timer_update(struct kbase_device *kbdev); */ int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev); -#endif /* KBASE_PM_RUNTIME */ - -#endif +#endif /* _KBASE_CSF_FIRMWARE_H_ */ diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware_no_mali.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware_no_mali.c index 0ac6ad394b2d..c8c57862a295 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware_no_mali.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_firmware_no_mali.c @@ -221,6 +221,12 @@ static int invent_capabilities(struct kbase_device *kbdev) iface->group_num = ARRAY_SIZE(interface->csg); iface->group_stride = 0; + iface->prfcnt_features = 0; + + /* Set METATA_SIZE in GLB_PRFCNT_FEATURES. Always equal to one block. */ + WARN_ON((KBASE_DUMMY_MODEL_BLOCK_SIZE >> 8) > 0xf); + iface->prfcnt_features |= ((KBASE_DUMMY_MODEL_BLOCK_SIZE >> 8) & 0xf); + iface->groups = kcalloc(iface->group_num, sizeof(*iface->groups), GFP_KERNEL); if (iface->groups == NULL) return -ENOMEM; @@ -584,11 +590,22 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) } /* Update shader core allocation enable mask */ + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + if (kbase_io_is_gpu_powered(kbdev)) + kbase_reg_write64(kbdev, GPU_GOVERNOR_ENUM(GOV_CORE_MASK), + kbase_pm_ca_get_gov_core_mask(kbdev)); + enable_endpoints_global(fw_io, core_mask); set_shader_poweroff_timer(fw_io); set_timeout_global(fw_io, kbase_csf_timeout_get(kbdev)); + /* Do not modify PRFCNT bits of GLB_ACK_IRQ_MASK. */ + ack_irq_mask |= + (kbase_csf_fw_io_global_input_read(&kbdev->csf.fw_io, GLB_ACK_IRQ_MASK) & + (GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK | GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK | + GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK | GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK)); + /* Unmask the interrupts */ kbase_csf_fw_io_global_write(fw_io, GLB_ACK_IRQ_MASK, ack_irq_mask); @@ -610,14 +627,32 @@ exit: */ static int global_init_on_boot(struct kbase_device *const kbdev) { - unsigned long flags; + unsigned long flags, scheduler_lock_flags; u64 core_mask; int ret = 0; u32 request_mask = CSF_GLB_REQ_CFG_MASK; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + core_mask = kbase_pm_ca_get_core_mask(kbdev); + kbdev->csf.firmware_hctl_core_pwr = kbase_pm_no_mcu_core_pwroff(kbdev); + + /* Enable the HWC context for the case of MCU controlling the shader core power. + * HWC context will be left enabled throughout the MCU power cycles, unless + * there is a GPU reset. + * + * In case of GPU reset, HWC needs to be disabled to ensure + * the correct HWC backend functionality. The context then gets reenabled during + * the reboot. + */ + if (!kbdev->csf.firmware_hctl_core_pwr) { + kbase_csf_scheduler_spin_lock(kbdev, &scheduler_lock_flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, scheduler_lock_flags); + kbdev->pm.backend.hwcnt_disabled = false; + kbdev->pm.backend.hwcnt_desired = true; + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); global_init(kbdev, core_mask); @@ -629,10 +664,25 @@ static int global_init_on_boot(struct kbase_device *const kbdev) void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev, u64 core_mask) { + unsigned long flags; + lockdep_assert_held(&kbdev->hwaccess_lock); kbdev->csf.glb_init_request_pending = true; kbdev->csf.firmware_hctl_core_pwr = kbase_pm_no_mcu_core_pwroff(kbdev); + if (!kbdev->csf.firmware_hctl_core_pwr) { + /* The HWC context reenabling is needed if: + * - shader core power control was previously under host. + * - context was disabled when preparing for a GPU reset. + */ + kbdev->pm.backend.hwcnt_desired = true; + if (kbdev->pm.backend.hwcnt_disabled) { + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + kbdev->pm.backend.hwcnt_disabled = false; + } + } global_init(kbdev, core_mask); } @@ -653,9 +703,6 @@ void kbase_csf_firmware_update_core_attr(struct kbase_device *kbdev, bool update struct kbase_csf_fw_io *fw_io = &kbdev->csf.fw_io; unsigned long flags, fw_io_flags; - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) - core_mask = U64_MAX; - lockdep_assert_held(&kbdev->hwaccess_lock); kbase_csf_scheduler_spin_lock(kbdev, &flags); @@ -688,10 +735,16 @@ static void kbase_csf_firmware_reload_worker(struct work_struct *work) container_of(work, struct kbase_device, csf.firmware_reload_work); unsigned long flags; + kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(&kbdev->hwcnt_gpu_iface); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); /* Reboot the firmware */ kbase_csf_firmware_enable_mcu(kbdev); + kbase_hwcnt_backend_csf_set_hw_availability(&kbdev->hwcnt_gpu_iface, + kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); /* Tell MCU state machine to transit to next state */ kbdev->csf.firmware_reloaded = true; kbase_pm_update_state(kbdev); @@ -709,6 +762,10 @@ void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) queue_work(system_wq, &kbdev->csf.firmware_reload_work); } else { kbase_csf_firmware_enable_mcu(kbdev); + kbase_hwcnt_backend_csf_set_hw_availability( + &kbdev->hwcnt_gpu_iface, kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, kbdev->pm.debug_core_mask); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); kbdev->csf.firmware_reloaded = true; } } @@ -721,7 +778,11 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) if (unlikely(!kbdev->csf.firmware_inited)) return; - + kbase_hwcnt_backend_csf_set_hw_availability(&kbdev->hwcnt_gpu_iface, + kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); + kbase_hwcnt_backend_csf_on_after_mcu_on(&kbdev->hwcnt_gpu_iface); /* Tell MCU state machine to transit to next state */ kbdev->csf.firmware_reloaded = true; kbase_pm_update_state(kbdev); @@ -955,6 +1016,15 @@ u32 kbase_csf_firmware_reset_mcu_core_pwroff_time(struct kbase_device *kbdev) return kbase_csf_firmware_set_mcu_core_pwroff_time(kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_NS); } +static void kbasep_hwcnt_init_on_boot(struct kbase_device *kbdev) +{ + kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(&kbdev->hwcnt_gpu_iface); + kbase_hwcnt_backend_csf_set_hw_availability(&kbdev->hwcnt_gpu_iface, + kbdev->gpu_props.curr_config.l2_slices, + kbdev->gpu_props.curr_config.shader_present, + kbdev->pm.debug_core_mask); +} + int kbase_csf_firmware_early_init(struct kbase_device *kbdev) { kbdev->csf.num_doorbells = CSF_NUM_DOORBELL_MAX; @@ -987,10 +1057,9 @@ int kbase_csf_firmware_late_init(struct kbase_device *kbdev) u32 no_modifier = 0; kbdev->csf.gpu_idle_hysteresis_ns = FIRMWARE_IDLE_HYSTERESIS_TIME_NS; -#ifdef KBASE_PM_RUNTIME - if (kbase_pm_gpu_sleep_allowed(kbdev)) + if (IS_ENABLED(CONFIG_PM) && kbase_pm_gpu_sleep_allowed(kbdev)) kbdev->csf.gpu_idle_hysteresis_ns /= FIRMWARE_IDLE_HYSTERESIS_GPU_SLEEP_SCALER; -#endif + WARN_ON(!kbdev->csf.gpu_idle_hysteresis_ns); kbdev->csf.gpu_idle_dur_count = convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &no_modifier); @@ -1041,6 +1110,8 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) kbase_csf_fw_io_init(&kbdev->csf.fw_io, kbdev); + kbasep_hwcnt_init_on_boot(kbdev); + ret = invent_capabilities(kbdev); if (ret != 0) goto error; @@ -1273,7 +1344,6 @@ void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL), MCU_CONTROL_REQ_AUTO); } -#ifdef KBASE_PM_RUNTIME void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev) { struct kbase_csf_fw_io *fw_io = &kbdev->csf.fw_io; @@ -1299,7 +1369,6 @@ bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev) kbase_csf_firmware_mcu_halted(kbdev)); } -#endif bool kbase_csf_firmware_mcu_halt_req_complete(struct kbase_device *kbdev) { @@ -1467,8 +1536,7 @@ int kbase_csf_firmware_mcu_shared_mapping_init(struct kbase_device *kbdev, unsig if (!page_list) goto page_list_alloc_error; - ret = kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, - phys, false, NULL); + ret = kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, NULL); if (ret <= 0) goto phys_mem_pool_alloc_error; @@ -1516,8 +1584,7 @@ va_region_add_error: va_region_alloc_error: vunmap(cpu_addr); vmap_error: - kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, phys, - false, false); + kbase_mem_pool_free_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, false); phys_mem_pool_alloc_error: kfree(page_list); @@ -1546,16 +1613,14 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev, } if (csf_mapping->phys) { - kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], - csf_mapping->num_pages, csf_mapping->phys, false, false); + kbase_mem_pool_free_pages(&kbdev->fw_mem_pools.small, csf_mapping->num_pages, + csf_mapping->phys, false, false); } vunmap(csf_mapping->cpu_addr); kfree(csf_mapping->phys); } -#ifdef KBASE_PM_RUNTIME - void kbase_csf_firmware_soi_update(struct kbase_device *kbdev) { } @@ -1568,5 +1633,3 @@ int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbd { return 0; } - -#endif /* KBASE_PM_RUNTIME */ diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.c index 6e20f9dfe302..a2ece518bd4a 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.c @@ -22,7 +22,6 @@ #include "mali_kbase.h" #include "mali_kbase_csf_fw_io.h" #include -#include "mali_kbase_io.h" #include static inline u32 input_page_read(const u32 *const input, const u32 offset) @@ -89,7 +88,7 @@ static inline void input_page_partial_write64(u64 *const input, const u32 offset void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io, struct kbase_device *kbdev) { spin_lock_init(&fw_io->lock); - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + bitmap_zero(fw_io->status, KBASEP_FW_IO_STATUS_NUM_BITS); fw_io->kbdev = kbdev; } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_init); @@ -358,19 +357,19 @@ KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_read); void kbase_csf_fw_io_set_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - kbase_io_set_status(fw_io->kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + set_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_set_status_gpu_suspended); void kbase_csf_fw_io_clear_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - kbase_io_clear_status(fw_io->kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + clear_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_clear_status_gpu_suspended); bool kbase_csf_fw_io_check_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - return kbase_io_test_status(fw_io->kbdev, KBASE_IO_STATUS_GPU_SUSPENDED); + return test_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_check_status_gpu_suspended); diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.h index a46400484840..ba6b6bcc2800 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io.h @@ -31,6 +31,17 @@ /** The wait completed because the GPU was lost. */ #define KBASE_CSF_FW_IO_WAIT_GPU_LOST 1 +/** + * enum kbasep_csf_fw_io_status_bits - Status bits for firmware I/O interface. + * + * @KBASEP_FW_IO_STATUS_GPU_SUSPENDED: The GPU is suspended. + * @KBASEP_FW_IO_STATUS_NUM_BITS: Number of bits used to encode the status. + */ +enum kbasep_csf_fw_io_status_bits { + KBASEP_FW_IO_STATUS_GPU_SUSPENDED = 0, + KBASEP_FW_IO_STATUS_NUM_BITS, +}; + /** * struct kbasep_csf_fw_io_stream_pages - Addresses to CS I/O pages. * @@ -72,11 +83,13 @@ struct kbasep_csf_fw_io_pages { * struct kbase_csf_fw_io - Manager of firmware input/output interface. * * @lock: Mutex to serialize access to the interface. + * @status: Internal status of the MCU interface. * @pages: Addresses to FW I/O pages * @kbdev: Pointer to the instance of a GPU platform device that implements a CSF interface. */ struct kbase_csf_fw_io { spinlock_t lock; + DECLARE_BITMAP(status, KBASEP_FW_IO_STATUS_NUM_BITS); struct kbasep_csf_fw_io_pages pages; struct kbase_device *kbdev; }; @@ -169,7 +182,8 @@ void kbase_csf_fw_io_pages_term(struct kbase_csf_fw_io *fw_io, u32 group_num); */ static inline int kbase_csf_fw_io_open(struct kbase_csf_fw_io *fw_io, unsigned long *flags) { - if (kbase_io_test_status(fw_io->kbdev, KBASE_IO_STATUS_GPU_SUSPENDED)) + if (test_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status) || + kbase_io_is_aw_removed(fw_io->kbdev)) return -KBASE_CSF_FW_IO_WAIT_GPU_LOST; spin_lock_irqsave(&fw_io->lock, *flags); @@ -410,8 +424,8 @@ bool kbase_csf_fw_io_check_status_gpu_suspended(struct kbase_csf_fw_io *fw_io); /** * kbase_csf_fw_io_wait_event_timeout() - Wait until condition gets true, timeout - * occurs or a GPU_SUSPENDED FW I/O status bit is set. The rest of the functionalities is equal - * to wait_event_timeout(). + * occurs, GPU_SUSPENDED FW I/O status bit is set, or AW was removed. + * The rest of the functionalities is equal to wait_event_timeout(). * * @fw_io: Firmware I/O manager. * @wq_head: The waitqueue to wait on. @@ -420,16 +434,20 @@ bool kbase_csf_fw_io_check_status_gpu_suspended(struct kbase_csf_fw_io *fw_io); * * Return: Remaining jiffies (at least 1) on success, * 0 on timeout, - * negative KBASE_CSF_FW_IO_WAIT_LOST error if GPU_SUSPENDED FW I/O status bit is set. + * negative KBASE_CSF_FW_IO_WAIT_LOST error + * if GPU_SUSPENDED FW I/O status bit is set + * or AW was removed. */ -#define kbase_csf_fw_io_wait_event_timeout(fw_io, wq_head, condition, timeout) \ - ({ \ - int __ret; \ - int __wait_remaining = wait_event_timeout( \ - wq_head, (condition) || kbase_csf_fw_io_check_status_gpu_suspended(fw_io), \ - timeout); \ - __ret = kbasep_csf_fw_io_handle_wait_result(fw_io, __wait_remaining); \ - __ret; \ +#define kbase_csf_fw_io_wait_event_timeout(fw_io, wq_head, condition, timeout) \ + ({ \ + int __ret; \ + int __wait_remaining = wait_event_timeout( \ + wq_head, \ + (condition) || kbase_csf_fw_io_check_status_gpu_suspended(fw_io) || \ + kbase_io_is_aw_removed((fw_io)->kbdev), \ + timeout); \ + __ret = kbasep_csf_fw_io_handle_wait_result(fw_io, __wait_remaining); \ + __ret; \ }) /** @@ -441,13 +459,17 @@ bool kbase_csf_fw_io_check_status_gpu_suspended(struct kbase_csf_fw_io *fw_io); * * Return: Remaining jiffies (at least 1) on success, * 0 on timeout, - * negative KBASE_CSF_FW_IO_WAIT_LOST error if GPU_SUSPENDED FW I/O status bit is set. + * negative KBASE_CSF_FW_IO_WAIT_LOST error + * if GPU_SUSPENDED FW I/O status bit is set + * or AW was removed. */ static inline int kbasep_csf_fw_io_handle_wait_result(struct kbase_csf_fw_io *fw_io, int wait_remaining) { - return kbase_csf_fw_io_check_status_gpu_suspended(fw_io) ? -KBASE_CSF_FW_IO_WAIT_GPU_LOST : - wait_remaining; + return (kbase_csf_fw_io_check_status_gpu_suspended(fw_io) || + kbase_io_is_aw_removed(fw_io->kbdev)) ? + -KBASE_CSF_FW_IO_WAIT_GPU_LOST : + wait_remaining; } #if IS_ENABLED(CONFIG_MALI_VALHALL_DEBUG) || IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io_no_mali.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io_no_mali.c index 9907c151df82..22da5bb78ae8 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io_no_mali.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_fw_io_no_mali.c @@ -87,7 +87,7 @@ static inline void input_page_partial_write64(u64 *const input, const u32 offset void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io, struct kbase_device *kbdev) { spin_lock_init(&fw_io->lock); - kbase_io_clear_status(kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + bitmap_zero(fw_io->status, KBASEP_FW_IO_STATUS_NUM_BITS); fw_io->kbdev = kbdev; } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_init); @@ -393,19 +393,19 @@ KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_read); void kbase_csf_fw_io_set_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - kbase_io_set_status(fw_io->kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + set_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_set_status_gpu_suspended); void kbase_csf_fw_io_clear_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - kbase_io_clear_status(fw_io->kbdev->io, KBASE_IO_STATUS_GPU_SUSPENDED); + clear_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_clear_status_gpu_suspended); bool kbase_csf_fw_io_check_status_gpu_suspended(struct kbase_csf_fw_io *fw_io) { - return kbase_io_test_status(fw_io->kbdev, KBASE_IO_STATUS_GPU_SUSPENDED); + return test_bit(KBASEP_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status); } KBASE_EXPORT_TEST_API(kbase_csf_fw_io_check_status_gpu_suspended); diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.c index 65db914b895d..e30fb051a43e 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.c @@ -1664,7 +1664,7 @@ static int kbase_kcpu_fence_force_signal_process(struct kbase_kcpu_command_queue #endif /* dma_fence refcount needs to be decreased to release it. */ - dma_fence_put(fence_info->fence); + kbase_fence_put(fence_info->fence); fence_info->fence = NULL; return ret; @@ -1700,6 +1700,7 @@ static void kcpu_force_signal_fence(struct kbase_kcpu_command_queue *kcpu_queue) /* set ETIMEDOUT error flag before signal the fence*/ dma_fence_set_error_helper(fence, -ETIMEDOUT); + kbase_fence_put(fence); /* force signal fence */ status = @@ -1860,6 +1861,8 @@ static int kbasep_kcpu_fence_signal_init(struct kbase_kcpu_command_queue *kcpu_q goto fd_flags_fail; } + __module_get(THIS_MODULE); + kcpu_fence->module = THIS_MODULE; fence->basep.fd = *fd; current_command->type = BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL; diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.h index e9aa7b1f4801..625ac06ef9c4 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_kcpu.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_mcu_shared_reg.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_mcu_shared_reg.c index c22bc1ac396d..1a8c7cedef75 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_mcu_shared_reg.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_mcu_shared_reg.c @@ -850,8 +850,8 @@ int kbase_csf_mcu_shared_regs_data_init(struct kbase_device *kbdev) if (!shared_regs->dummy_phys) return -ENOMEM; - if (kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], 1, - &shared_regs->dummy_phys[0], false, NULL) <= 0) + if (kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, 1, &shared_regs->dummy_phys[0], + false, NULL) <= 0) return -ENOMEM; shared_regs->dummy_phys_allocated = true; @@ -920,7 +920,7 @@ void kbase_csf_mcu_shared_regs_data_term(struct kbase_device *kbdev) if (shared_regs->dummy_phys_allocated) { struct page *page = as_page(shared_regs->dummy_phys[0]); - kbase_mem_pool_free(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, false); + kbase_mem_pool_free(&kbdev->fw_mem_pools.small, page, false); } kfree(shared_regs->dummy_phys); diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_registers.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_registers.h index deb1878a0e6b..c08f48f48ec7 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_registers.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_registers.h @@ -204,6 +204,7 @@ #define GLB_GROUP_STRIDE 0x0014 /* () Stride between CSG interfaces */ #define GLB_PRFCNT_SIZE 0x0018 /* () Size of CSF performance counters */ #define GLB_INSTR_FEATURES 0x001C /* () TRACE_POINT instrumentation. (csf >= 1.1.0) */ +#define GLB_PRFCNT_FEATURES 0x0020 /* () Performance counter features */ #define GROUP_CONTROL_0 0x1000 /* () CSG control and capabilities */ #define GROUP_CONTROL(n) (GROUP_CONTROL_0 + (n)*256) #define GROUP_CONTROL_REG(n, r) (GROUP_CONTROL(n) + GROUP_CONTROL_BLOCK_REG(r)) @@ -1982,6 +1983,16 @@ #define GLB_PRFCNT_CONFIG_SET_SELECT_SET(reg_val, value) \ (((reg_val) & ~GLB_PRFCNT_CONFIG_SET_SELECT_MASK) | \ (((value) << GLB_PRFCNT_CONFIG_SET_SELECT_SHIFT) & GLB_PRFCNT_CONFIG_SET_SELECT_MASK)) +#define GLB_PRFCNT_CONFIG_METADATA_ENABLE_SHIFT GPU_U(10) +#define GLB_PRFCNT_CONFIG_METADATA_ENABLE_MASK \ + (GPU_U(0x1) << GLB_PRFCNT_CONFIG_METADATA_ENABLE_SHIFT) +#define GLB_PRFCNT_CONFIG_METADATA_ENABLE_GET(reg_val) \ + (((reg_val)&GLB_PRFCNT_CONFIG_METADATA_ENABLE_MASK) >> \ + GLB_PRFCNT_CONFIG_METADATA_ENABLE_SHIFT) +#define GLB_PRFCNT_CONFIG_METADATA_ENABLE_SET(reg_val, value) \ + (~(~(reg_val) | GLB_PRFCNT_CONFIG_METADATA_ENABLE_MASK) | \ + (((value) << GLB_PRFCNT_CONFIG_METADATA_ENABLE_SHIFT) & \ + GLB_PRFCNT_CONFIG_METADATA_ENABLE_MASK)) /* GLB_PRFCNT_SIZE register */ #define GLB_PRFCNT_SIZE_HARDWARE_SIZE_SET_MOD(value) ((value) >> 8) @@ -2007,6 +2018,16 @@ ((GLB_PRFCNT_SIZE_FIRMWARE_SIZE_SET_MOD(value) << GLB_PRFCNT_SIZE_FIRMWARE_SIZE_SHIFT) & \ GLB_PRFCNT_SIZE_FIRMWARE_SIZE_MASK)) +/* GLB_PRFCNT_FEATURES register */ +#define GLB_PRFCNT_FEATURES_METADATA_SIZE_GET_MOD(value) (value << 8) +#define GLB_PRFCNT_FEATURES_METADATA_SIZE_SHIFT GPU_U(0) +#define GLB_PRFCNT_FEATURES_METADATA_SIZE_MASK \ + (GPU_U(0xF) << GLB_PRFCNT_FEATURES_METADATA_SIZE_SHIFT) +#define GLB_PRFCNT_FEATURES_METADATA_SIZE_GET(reg_val) \ + (GLB_PRFCNT_FEATURES_METADATA_SIZE_GET_MOD( \ + ((reg_val)&GLB_PRFCNT_FEATURES_METADATA_SIZE_MASK) >> \ + GLB_PRFCNT_FEATURES_METADATA_SIZE_SHIFT)) + /* GLB_DEBUG_REQ register */ #define GLB_DEBUG_REQ_DEBUG_RUN_SHIFT GPU_U(23) #define GLB_DEBUG_REQ_DEBUG_RUN_MASK (GPU_U(0x1) << GLB_DEBUG_REQ_DEBUG_RUN_SHIFT) diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_reset_gpu.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_reset_gpu.c index 03fdf60ce1b5..1fa97035a8dd 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_reset_gpu.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_reset_gpu.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -243,7 +243,6 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)), kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK)), kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK))); -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { dev_err(kbdev->dev, " PWR_IRQ_RAWSTAT=0x%08x", kbase_reg_read32(kbdev, HOST_POWER_ENUM(PWR_IRQ_RAWSTAT))); @@ -252,7 +251,6 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) dev_err(kbdev->dev, " PWR_STATUS=0x%016llx", kbase_reg_read64(kbdev, HOST_POWER_ENUM(PWR_STATUS))); } -#endif if (kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(14, 10, 0)) { dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(PWR_OVERRIDE0)), @@ -264,7 +262,6 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(TILER_CONFIG))); } -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { u32 domain_status; @@ -276,7 +273,6 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) dev_err(kbdev->dev, " L2_PWR_STATUS=0x%05llx", kbase_reg_read64(kbdev, HOST_POWER_ENUM(PWR_CMDARG))); } -#endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } @@ -426,13 +422,17 @@ static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, bool firmware_ini spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->csf.firmware_reload_needed = false; + /* Prevent MCU enable before FW data is reloaded */ + kbdev->pm.backend.in_reset = true; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); cancel_work_sync(&kbdev->csf.firmware_reload_work); - dev_dbg(kbdev->dev, "Disable GPU hardware counters.\n"); - /* This call will block until counters are disabled. */ - kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + { + dev_dbg(kbdev->dev, "Disable GPU hardware counters.\n"); + /* This call will block until counters are disabled. */ + kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + } ret = kbase_csf_reset_gpu_once(kbdev, firmware_inited, silent); @@ -488,13 +488,12 @@ static void kbase_csf_reset_gpu_worker(struct work_struct *data) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); firmware_inited = kbdev->csf.firmware_inited; -#ifdef KBASE_PM_RUNTIME - gpu_sleep_mode_active = kbdev->pm.backend.gpu_sleep_mode_active; -#endif + if (IS_ENABLED(CONFIG_PM)) + gpu_sleep_mode_active = kbdev->pm.backend.gpu_sleep_mode_active; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (unlikely(gpu_sleep_mode_active)) { -#ifdef KBASE_PM_RUNTIME + if (unlikely(gpu_sleep_mode_active) && IS_ENABLED(CONFIG_PM)) { /* As prior to GPU reset all on-slot groups are suspended, * need to wake up the MCU from sleep. * No pm active reference is taken here since GPU is in sleep @@ -508,7 +507,6 @@ static void kbase_csf_reset_gpu_worker(struct work_struct *data) kbase_pm_unlock(kbdev); err = kbase_csf_reset_gpu_now(kbdev, firmware_inited, silent); -#endif } else if (!kbase_pm_context_active_handle_suspend( kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { err = kbase_csf_reset_gpu_now(kbdev, firmware_inited, silent); diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.c index bda9bf1ceafa..ca7db6074bab 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.c @@ -277,6 +277,9 @@ static u64 drain_gpu_metrics_trace_buffer(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->csf.scheduler.gpu_metrics_lock); + if (kbase_io_is_aw_removed(kbdev)) + return 0; + kbase_backend_get_gpu_time_norequest(kbdev, NULL, &system_time, NULL); /* CPU time value that was used to derive the parameters for time conversion, * was retrieved from ktime_get_raw_ts64(). But the tracing subsystem would use @@ -503,7 +506,6 @@ static void schedule_actions_trigger_df(struct kbase_device *kbdev, struct kbase #endif } -#ifdef KBASE_PM_RUNTIME /** * wait_for_scheduler_to_exit_sleep() - Wait for Scheduler to exit the * sleeping state. @@ -519,7 +521,11 @@ static void schedule_actions_trigger_df(struct kbase_device *kbdev, struct kbase static int wait_for_scheduler_to_exit_sleep(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; +#if IS_ENABLED(CONFIG_PM) int autosuspend_delay = kbdev->dev->power.autosuspend_delay; +#else + int autosuspend_delay = 0; +#endif unsigned int sleep_exit_wait_time; long remaining; int ret = 0; @@ -578,7 +584,7 @@ static int force_scheduler_to_exit_sleep(struct kbase_device *kbdev) lockdep_assert_held(&scheduler->lock); WARN_ON(scheduler->state != SCHED_SLEEPING); - WARN_ON(!kbdev->pm.backend.gpu_sleep_mode_active); + WARN_ON(!kbdev->pm.backend.gpu_sleep_mode_active && !kbase_io_is_aw_removed(kbdev)); kbase_pm_lock(kbdev); ret = kbase_pm_force_mcu_wakeup_after_sleep(kbdev); @@ -626,7 +632,6 @@ out: return ret; } -#endif /** * tick_timer_callback() - Callback function for the scheduling tick hrtimer @@ -1061,7 +1066,6 @@ static int scheduler_pm_active_handle_suspend(struct kbase_device *kbdev, return ret; } -#ifdef KBASE_PM_RUNTIME /** * scheduler_pm_active_after_sleep() - Acquire the PM reference count for * Scheduler @@ -1084,7 +1088,6 @@ static int scheduler_pm_active_after_sleep(struct kbase_device *kbdev) return scheduler_pm_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE, true); } -#endif /** * scheduler_pm_idle() - Release the PM reference count held by Scheduler @@ -1102,7 +1105,6 @@ static inline void scheduler_pm_idle(struct kbase_device *kbdev) kbase_pm_context_idle(kbdev); } -#ifdef KBASE_PM_RUNTIME /** * scheduler_pm_idle_before_sleep() - Release the PM reference count and * trigger the transition to sleep state. @@ -1126,7 +1128,6 @@ static void scheduler_pm_idle_before_sleep(struct kbase_device *kbdev) scheduler_pm_idle(kbdev); } -#endif static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) { @@ -1149,11 +1150,9 @@ static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) HRTIMER_MODE_REL_SOFT); } #endif - } else { -#ifdef KBASE_PM_RUNTIME + } else if (IS_ENABLED(CONFIG_PM)) { dev_dbg(kbdev->dev, "Re-activating the Scheduler out of sleep"); ret = scheduler_pm_active_after_sleep(kbdev); -#endif } if (ret) { @@ -1178,19 +1177,21 @@ static int scheduler_suspend(struct kbase_device *kbdev) lockdep_assert_held(&scheduler->lock); if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) { -#if KBASE_PM_RUNTIME - int ret; + if (IS_ENABLED(CONFIG_PM)) { + int ret; - /* If GPU-level suspend is not supported then all on-slot CSGs - * will be suspended one at a time. Disable Sleep-on-Idle to - * avoid repeated cycles of FW going to sleep and resuming. - * GLB_IDLE timer (and Sleep-on-Idle) would otherwise be - * disabled just before the MCU transitions out of HALT state. - */ - ret = kbase_csf_firmware_soi_disable_on_scheduler_suspend(kbdev); - if (ret) - return ret; -#endif /* KBASE_PM_RUNTIME */ + /* If GPU-level suspend is not supported then all on-slot CSGs + * will be suspended one at a time. Disable Sleep-on-Idle to + * avoid repeated cycles of FW going to sleep and resuming. + * GLB_IDLE timer (and Sleep-on-Idle) would otherwise be + * disabled just before the MCU transitions out of HALT state. + */ + { + ret = kbase_csf_firmware_soi_disable_on_scheduler_suspend(kbdev); + if (ret) + return ret; + } + } dev_dbg(kbdev->dev, "Suspending the Scheduler"); scheduler_pm_idle(kbdev); scheduler->state = SCHED_SUSPENDED; @@ -2321,7 +2322,7 @@ static void process_cs_pending_events(struct kbase_csf_fw_io *fw_io, u32 group_i } /** - * save_slot_cs() - Save the state for blocked GPU command queue. + * save_cs_wait_state() - Save the state for blocked GPU command queue. * * @fw_io: Pointer to FW I/O manager. * @group_id: GPU on-slot CSG index. @@ -2334,7 +2335,8 @@ static void process_cs_pending_events(struct kbase_csf_fw_io *fw_io, u32 group_i * * Return: true if the queue is blocked on a sync wait operation. */ -static bool save_slot_cs(struct kbase_csf_fw_io *fw_io, u32 group_id, struct kbase_queue *queue) +static bool save_cs_wait_state(struct kbase_csf_fw_io *fw_io, u32 group_id, + struct kbase_queue *queue) { u32 stream_id = queue->csi_index; u32 status; @@ -2388,8 +2390,6 @@ static bool save_slot_cs(struct kbase_csf_fw_io *fw_io, u32 group_id, struct kba queue->status_wait = 0; } - process_cs_pending_events(fw_io, group_id, queue); - return is_waiting; } @@ -2792,31 +2792,35 @@ static void save_csg_slot(struct kbase_queue_group *group) for (i = 0; i < max_streams; i++) update_hw_active(group->bound_queues[i], false); #endif /* CONFIG_MALI_VALHALL_NO_MALI */ - for (i = 0; idle && i < max_streams; i++) { + for (i = 0; i < max_streams; i++) { struct kbase_queue *const queue = group->bound_queues[i]; if (!queue || !queue->enabled) continue; - if (save_slot_cs(&kbdev->csf.fw_io, group->csg_nr, queue)) { - /* sync_wait is only true if the queue is blocked on - * a CQS and not a scoreboard. - */ - if (queue->blocked_reason != CS_STATUS_BLOCKED_ON_SB_WAIT) - sync_wait = true; - } else { - /* Need to confirm if ringbuffer of the GPU - * queue is empty or not. A race can arise - * between the flush of GPU queue and suspend - * of CSG. If a queue is flushed after FW has - * set the IDLE bit in CSG_STATUS_STATE, then - * Scheduler will incorrectly consider CSG - * as idle. And there may not be any further - * flush call for the GPU queue, which would - * have de-idled the CSG. - */ - idle = confirm_cmd_buf_empty(queue); + if (idle) { + if (save_cs_wait_state(&kbdev->csf.fw_io, group->csg_nr, queue)) { + /* sync_wait is only true if the queue is blocked on + * a CQS and not a scoreboard. + */ + if (queue->blocked_reason != CS_STATUS_BLOCKED_ON_SB_WAIT) + sync_wait = true; + } else { + /* Need to confirm if ringbuffer of the GPU + * queue is empty or not. A race can arise + * between the flush of GPU queue and suspend + * of CSG. If a queue is flushed after FW has + * set the IDLE bit in CSG_STATUS_STATE, then + * Scheduler will incorrectly consider CSG + * as idle. And there may not be any further + * flush call for the GPU queue, which would + * have de-idled the CSG. + */ + idle = confirm_cmd_buf_empty(queue); + } } + + process_cs_pending_events(&kbdev->csf.fw_io, group->csg_nr, queue); } if (idle) { @@ -3028,7 +3032,8 @@ static void program_csg_slot(struct kbase_queue_group *group, s8 slot, u8 prio) const u64 compute_mask = shader_core_mask & group->compute_mask; const u64 fragment_mask = shader_core_mask & group->fragment_mask; const u64 tiler_mask = tiler_core_mask & group->tiler_mask; - const u64 neural_mask = shader_core_mask & group->neural_mask; + const u64 neural_mask = shader_core_mask & group->neural_mask & + kbdev->csf.neural_allowed_mask; const u8 neural_max = min(kbdev->gpu_props.num_cores, group->neural_max); const u8 comp_pri_threshold = min_t(u8, group->comp_pri_threshold, (u8)COMP_PRI_THRESHOLD_MAX); @@ -3116,6 +3121,13 @@ static void program_csg_slot(struct kbase_queue_group *group, s8 slot, u8 prio) goto skip_fw; } + if (kbdev->csf.neural_allowed_mask != UINT64_MAX) + dev_warn_once( + kbdev->dev, + "neural_allowed_mask module param passed in as 0x%llx. Writing restricted neural_mask 0x%llx to CSG_ALLOW_NEURAL. shader_core_mask 0x%llx group->neural_mask 0x%llx", + kbdev->csf.neural_allowed_mask, neural_mask, shader_core_mask, + group->neural_mask); + /* Endpoint programming for CSG */ kbase_csf_fw_io_group_write(fw_io, slot, CSG_ALLOW_COMPUTE_LO, compute_mask & U32_MAX); kbase_csf_fw_io_group_write(fw_io, slot, CSG_ALLOW_COMPUTE_HI, compute_mask >> 32); @@ -3356,7 +3368,6 @@ void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group) on_slot = kbasep_csf_scheduler_group_is_on_slot_locked(group); -#ifdef KBASE_PM_RUNTIME /* If the queue group is on slot and Scheduler is in SLEEPING state, * then we need to wake up the Scheduler to exit the sleep state rather * than waiting for the runtime suspend or power down of GPU. @@ -3364,7 +3375,7 @@ void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group) * thread and it has been seen that certain Apps can destroy groups at * random points and not necessarily when the App is exiting. */ - if (on_slot && (scheduler->state == SCHED_SLEEPING)) { + if (IS_ENABLED(CONFIG_PM) && on_slot && (scheduler->state == SCHED_SLEEPING)) { scheduler_wakeup(kbdev, true); /* Wait for MCU firmware to start running */ @@ -3380,7 +3391,7 @@ void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group) wait_for_termination = false; } } -#endif + if (!on_slot) { sched_evict_group(group, false, true); } else { @@ -5144,7 +5155,6 @@ static bool scheduler_idle_suspendable(struct kbase_device *kbdev) return suspend; } -#ifdef KBASE_PM_RUNTIME /** * scheduler_sleep_on_idle - Put the Scheduler in sleeping state on GPU * becoming idle. @@ -5168,7 +5178,6 @@ static void scheduler_sleep_on_idle(struct kbase_device *kbdev) scheduler->state = SCHED_SLEEPING; KBASE_KTRACE_ADD(kbdev, SCHED_SLEEPING, NULL, scheduler->state); } -#endif /** * scheduler_suspend_on_idle - Put the Scheduler in suspended state on GPU @@ -5213,6 +5222,11 @@ static void gpu_idle_worker(struct kbase_device *kbdev) WARN_ON_ONCE(atomic_read(&scheduler->pending_gpu_idle_work) == 0); + if (kbase_io_is_aw_removed(kbdev)) { + dev_info(kbdev->dev, "%s(): aborting as AW is no longer connected", __func__); + goto exit; + } + KBASE_KTRACE_ADD(kbdev, SCHEDULER_GPU_IDLE_WORKER_START, NULL, 0u); #define __ENCODE_KTRACE_INFO(reset, idle, all_suspend) \ @@ -5239,12 +5253,11 @@ static void gpu_idle_worker(struct kbase_device *kbdev) if (scheduler_is_idle_suspendable) { KBASE_KTRACE_ADD(kbdev, SCHEDULER_GPU_IDLE_WORKER_HANDLING_START, NULL, kbase_csf_ktrace_gpu_cycle_cnt(kbdev)); -#ifdef KBASE_PM_RUNTIME - if (kbase_pm_gpu_sleep_allowed(kbdev) && + + if (IS_ENABLED(CONFIG_PM) && kbase_pm_gpu_sleep_allowed(kbdev) && kbase_csf_scheduler_get_nr_active_csgs(kbdev)) scheduler_sleep_on_idle(kbdev); else -#endif all_groups_suspended = scheduler_suspend_on_idle(kbdev); KBASE_KTRACE_ADD(kbdev, SCHEDULER_GPU_IDLE_WORKER_HANDLING_END, NULL, 0u); @@ -5758,8 +5771,16 @@ static bool can_skip_scheduling(struct kbase_device *kbdev) if (scheduler->state == SCHED_SUSPENDED) return true; -#ifdef KBASE_PM_RUNTIME - if (scheduler->state == SCHED_SLEEPING) { + /* + * Schduler should always be in SUSPENDED state when + * AW is removed. + */ + if (kbase_io_is_aw_removed(kbdev)) { + dev_warn(kbdev->dev, "Skipping scheduling as AW is no longer connected"); + return true; + } + + if (IS_ENABLED(CONFIG_PM) && scheduler->state == SCHED_SLEEPING) { unsigned long flags; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -5780,7 +5801,6 @@ static bool can_skip_scheduling(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return true; } -#endif return false; } @@ -5834,7 +5854,7 @@ exit_no_schedule_unlock: static void schedule_on_tick(struct kbase_device *kbdev) { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; - + struct kbase_csf_sched_heap_reclaim_mgr *mgr = &kbdev->csf.scheduler.reclaim_mgr; int err = kbase_reset_gpu_try_prevent(kbdev); /* Regardless of whether reset failed or is currently happening, exit * early @@ -5871,6 +5891,8 @@ static void schedule_on_tick(struct kbase_device *kbdev) } scheduler->state = SCHED_INACTIVE; + if (atomic_read(&mgr->unused_pages)) + kbase_csf_tiler_heap_reclaim_unused_pages(kbdev, HEAP_RECLAIM_SCENARIO_SCHEDULER); mutex_unlock(&scheduler->lock); KBASE_KTRACE_ADD(kbdev, SCHED_INACTIVE, NULL, scheduler->state); kbase_reset_gpu_allow(kbdev); @@ -5903,8 +5925,10 @@ static int suspend_active_queue_groups(struct kbase_device *kbdev, unsigned long suspend_queue_group(group); set_bit(slot_num, slot_mask); } - ret = wait_csg_slots_suspend(kbdev, slot_mask); } + + ret = wait_csg_slots_suspend(kbdev, slot_mask); + return ret; } @@ -6083,15 +6107,14 @@ static void scheduler_inner_reset(struct kbase_device *kbdev) scheduler->num_active_address_spaces | (((u64)scheduler->total_runnable_grps) << 32)); -#ifdef KBASE_PM_RUNTIME - if (scheduler->state == SCHED_SLEEPING) { + if (IS_ENABLED(CONFIG_PM) && scheduler->state == SCHED_SLEEPING) { #if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) hrtimer_cancel(&scheduler->gpu_metrics_timer); #endif scheduler->state = SCHED_SUSPENDED; KBASE_KTRACE_ADD(kbdev, SCHED_SUSPENDED, NULL, scheduler->state); } -#endif + mutex_unlock(&scheduler->lock); } @@ -6241,8 +6264,7 @@ int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, on_slot = kbasep_csf_scheduler_group_is_on_slot_locked(group); -#ifdef KBASE_PM_RUNTIME - if (on_slot && (scheduler->state == SCHED_SLEEPING)) { + if (IS_ENABLED(CONFIG_PM) && on_slot && (scheduler->state == SCHED_SLEEPING)) { if (wait_for_scheduler_to_exit_sleep(kbdev)) { dev_warn( kbdev->dev, @@ -6265,7 +6287,7 @@ int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, */ on_slot = kbasep_csf_scheduler_group_is_on_slot_locked(group); } -#endif + if (on_slot) { DECLARE_BITMAP(slot_mask, BASEP_QUEUE_GROUP_MAX) = { 0 }; @@ -6812,7 +6834,6 @@ event_wait_add_failed: void kbase_csf_scheduler_context_term(struct kbase_context *kctx) { - kbase_ctx_sched_remove_ctx(kctx); #if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) gpu_metrics_ctx_term(kctx); #endif /* CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD */ @@ -6984,6 +7005,12 @@ static int kbase_csf_scheduler_kthread(void *data) while (atomic_read(&scheduler->pending_gpu_idle_work) > 0) gpu_idle_worker(kbdev); + /* Drain pending GPU suspend work */ + if (atomic_read(&scheduler->pending_runtime_suspend_work) == true) { + kbdev->pm.runtime_suspend_result = kbase_pm_handle_runtime_suspend(kbdev); + atomic_set(&scheduler->pending_runtime_suspend_work, false); + } + /* Drain pending GPU power off work */ if (atomic_cmpxchg(&scheduler->pending_power_off_work, true, false) == true) kbase_pm_handle_gpu_poweroff_wait_work(kbdev); @@ -7097,6 +7124,7 @@ int kbase_csf_scheduler_early_init(struct kbase_device *kbdev) atomic_set(&scheduler->pending_tick_work, false); atomic_set(&scheduler->pending_tock_work, false); atomic_set(&scheduler->pending_gpu_idle_work, 0); + atomic_set(&scheduler->pending_runtime_suspend_work, false); atomic_set(&scheduler->pending_power_off_work, false); return kbase_csf_tiler_heap_reclaim_mgr_init(kbdev); @@ -7240,11 +7268,22 @@ int kbase_csf_scheduler_pm_suspend_no_lock(struct kbase_device *kbdev) return -EBUSY; #endif -#ifdef KBASE_PM_RUNTIME + if (kbase_io_is_aw_removed(kbdev)) { + dev_warn(kbdev->dev, "Suspending scheduler(AW_REMOVED), state: %d", + scheduler->state); + if (scheduler->state != SCHED_SUSPENDED && scheduler->state != SCHED_SLEEPING) + scheduler_pm_idle(kbdev); + scheduler->state = SCHED_SUSPENDED; +#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) + hrtimer_cancel(&scheduler->gpu_metrics_timer); +#endif + return 0; + } + /* If scheduler is in sleeping state, then MCU needs to be activated * to suspend CSGs. */ - if (scheduler->state == SCHED_SLEEPING) { + if (IS_ENABLED(CONFIG_PM) && scheduler->state == SCHED_SLEEPING) { dev_info(kbdev->dev, "Activating MCU out of sleep on system suspend"); result = force_scheduler_to_exit_sleep(kbdev); if (result) { @@ -7252,7 +7291,7 @@ int kbase_csf_scheduler_pm_suspend_no_lock(struct kbase_device *kbdev) goto exit; } } -#endif + if (scheduler->state != SCHED_SUSPENDED) { result = suspend_active_groups_on_powerdown(kbdev, true); if (result) { @@ -7267,6 +7306,17 @@ int kbase_csf_scheduler_pm_suspend_no_lock(struct kbase_device *kbdev) } exit: + if (result && kbase_io_is_aw_removed(kbdev)) { + dev_warn(kbdev->dev, "Forcing scheduler suspend(AW_REMOVED), state: %d", + scheduler->state); + if (scheduler->state != SCHED_SUSPENDED && scheduler->state != SCHED_SLEEPING) + scheduler_pm_idle(kbdev); + scheduler->state = SCHED_SUSPENDED; +#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) + hrtimer_cancel(&scheduler->gpu_metrics_timer); +#endif + result = 0; + } return result; } @@ -7361,7 +7411,10 @@ static int scheduler_wait_mcu_active(struct kbase_device *kbdev, bool killable_w err = kbase_pm_wait_for_desired_state(kbdev); if (!err) { spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_ON); + if (kbase_io_is_aw_removed(kbdev)) + err = -ENODEV; + else + WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_ON); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } @@ -7380,7 +7433,6 @@ int kbase_csf_scheduler_wait_mcu_active(struct kbase_device *kbdev) KBASE_EXPORT_TEST_API(kbase_csf_scheduler_wait_mcu_active); -#ifdef KBASE_PM_RUNTIME int kbase_csf_scheduler_handle_runtime_suspend(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; @@ -7532,7 +7584,6 @@ void kbase_csf_scheduler_force_sleep(struct kbase_device *kbdev) scheduler_sleep_on_idle(kbdev); mutex_unlock(&scheduler->lock); } -#endif void kbase_csf_scheduler_force_wakeup(struct kbase_device *kbdev) { diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.h index 50d2f0fbd853..96e9e35b49db 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_scheduler.h @@ -610,7 +610,6 @@ static inline bool kbase_csf_scheduler_queue_has_trace(struct kbase_queue *queue return (queue->trace_buffer_size && queue->trace_buffer_base); } -#ifdef KBASE_PM_RUNTIME /** * kbase_csf_scheduler_reval_idleness_post_sleep() - Check GPU's idleness after * putting MCU to sleep state @@ -645,7 +644,6 @@ void kbase_csf_scheduler_reval_idleness_post_sleep(struct kbase_device *kbdev); * Return: 0 if all the CSGs were suspended, otherwise an error code. */ int kbase_csf_scheduler_handle_runtime_suspend(struct kbase_device *kbdev); -#endif /** * kbase_csf_scheduler_process_gpu_idle_event() - Process GPU idle IRQ @@ -697,7 +695,6 @@ u32 kbase_csf_scheduler_get_nr_active_csgs_locked(struct kbase_device *kbdev); */ void kbase_csf_scheduler_force_wakeup(struct kbase_device *kbdev); -#ifdef KBASE_PM_RUNTIME /** * kbase_csf_scheduler_force_sleep() - Forcefully put the Scheduler to sleeping * state. @@ -710,7 +707,6 @@ void kbase_csf_scheduler_force_wakeup(struct kbase_device *kbdev); * This function is only used for testing purpose. */ void kbase_csf_scheduler_force_sleep(struct kbase_device *kbdev); -#endif #endif /* _KBASE_CSF_SCHEDULER_H_ */ diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.c index ce0ecea13d16..8496d145f6fd 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.c @@ -34,6 +34,13 @@ /* Tiler heap reclaim scan (free) method size for limiting a scan run length */ #define HEAP_RECLAIM_SCAN_BATCH_SIZE (HEAP_SHRINKER_BATCH << 7) +/* + * Default setting for reclaiming offslot CSG's heap. + * Disabling reclaim with pages being 0. + */ +#define HEAP_RECLAIM_OFFSLOT_TIMEOUT_MS (30000) +#define HEAP_RECLAIM_OFFSLOT_PAGES (0) + static u8 get_kctx_highest_csg_priority(struct kbase_context *kctx) { u8 prio; @@ -76,6 +83,9 @@ static void detach_ctx_from_heap_reclaim_mgr(struct kbase_context *kctx) "Reclaim_mgr_detach: ctx_%d_%d, est_pages=0%u, freed_pages=%u", kctx->tgid, kctx->id, info->nr_est_unused_pages, info->nr_freed_pages); } + + /* 0 indicates that the kctx may have CSG on slot */ + kctx->offslot_ts = 0; } static void attach_ctx_to_heap_reclaim_mgr(struct kbase_context *kctx) @@ -98,8 +108,9 @@ static void attach_ctx_to_heap_reclaim_mgr(struct kbase_context *kctx) /* Accumulate the estimated pages to the manager total field */ atomic_add((int)info->nr_est_unused_pages, &scheduler->reclaim_mgr.unused_pages); - dev_dbg(kctx->kbdev->dev, "Reclaim_mgr_attach: ctx_%d_%d, est_count_pages=%u", kctx->tgid, - kctx->id, info->nr_est_unused_pages); + kctx->offslot_ts = ktime_get_raw_ns(); + dev_dbg(kctx->kbdev->dev, "Reclaim_mgr_attach [%llu]: ctx_%d_%d, est_count_pages=%u", + kctx->offslot_ts, kctx->tgid, kctx->id, info->nr_est_unused_pages); } void kbase_csf_tiler_heap_reclaim_sched_notify_grp_active(struct kbase_queue_group *group) @@ -187,15 +198,43 @@ void kbase_csf_tiler_heap_reclaim_sched_notify_grp_suspend(struct kbase_queue_gr } } -static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev) +unsigned long kbase_csf_tiler_heap_reclaim_unused_pages(struct kbase_device *kbdev, + enum heap_reclaim_scenario scenario) { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; struct kbase_csf_sched_heap_reclaim_mgr *const mgr = &scheduler->reclaim_mgr; - unsigned long total_freed_pages = 0; - int prio; + unsigned long total_freed_pages = 0, max_pages; + int prio, min_prio; + u64 now = 0, offslot_ts = 0; lockdep_assert_held(&scheduler->lock); + if (scenario == HEAP_RECLAIM_SCENARIO_SHRINKER) { + /* triggered by shrinker */ + max_pages = HEAP_RECLAIM_SCAN_BATCH_SIZE; + min_prio = KBASE_QUEUE_GROUP_PRIORITY_REALTIME; + } else if (scenario == HEAP_RECLAIM_SCENARIO_SCHEDULER) { + /* triggered by schedule_on_tick + * reclaim heap from CSGs with below conditions: + * 1.being offslot for a period + * 2.limit page numbers for each reclaim + * 3.skip RT kctx + */ + if (mgr->offslot_setting.pages == 0) { + dev_dbg(kbdev->dev, "HEAP_RECLAIM_SCENARIO_SCHEDULER is disabled"); + return 0; + } + max_pages = mgr->offslot_setting.pages; + min_prio = KBASE_QUEUE_GROUP_PRIORITY_HIGH; + now = ktime_get_raw_ns(); + offslot_ts = now > mgr->offslot_setting.timeout_ms * 1000000ULL ? + now - mgr->offslot_setting.timeout_ms * 1000000ULL : + 0; + } else { + dev_err(kbdev->dev, "Unknown heap reclaim scenario %d\n", scenario); + return 0; + } + if (scheduler->state != SCHED_SUSPENDED) { /* Clean and invalidate the L2 cache before reading from the heap contexts, * headers of the individual chunks and buffer descriptors. @@ -219,9 +258,7 @@ static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev) } for (prio = KBASE_QUEUE_GROUP_PRIORITY_LOW; - total_freed_pages < HEAP_RECLAIM_SCAN_BATCH_SIZE && - prio >= KBASE_QUEUE_GROUP_PRIORITY_REALTIME; - prio--) { + total_freed_pages < max_pages && prio >= min_prio; prio--) { struct kbase_csf_ctx_heap_reclaim_info *info, *tmp; u32 cnt_ctxs = 0; @@ -229,9 +266,26 @@ static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev) mgr_link) { struct kbase_context *kctx = container_of(info, struct kbase_context, csf.sched.heap_info); - u32 freed_pages = kbase_csf_tiler_heap_scan_kctx_unused_pages( + u32 freed_pages; + + if (scenario == HEAP_RECLAIM_SCENARIO_SCHEDULER) { + WARN_ON(kctx->offslot_ts == 0); + if (kctx->offslot_ts > offslot_ts) { + dev_dbg(kbdev->dev, + "Reclaim aborts from ctx %d_%d, prio %d, current time %llu - offslot time %llu = %llu", + kctx->tgid, kctx->id, prio, now, kctx->offslot_ts, + now - kctx->offslot_ts); + /* Skip following ctx as they are attached later */ + break; + } + } + + freed_pages = kbase_csf_tiler_heap_scan_kctx_unused_pages( kctx, info->nr_est_unused_pages); + dev_dbg(kbdev->dev, "Reclaim free heap pages for ctx %d_%d freed pages %u", + kctx->tgid, kctx->id, freed_pages); + if (freed_pages) { /* Remove the freed pages from the manager retained estimate. The * accumulated removals from the kctx should not exceed the kctx @@ -260,7 +314,7 @@ static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev) cnt_ctxs++; /* Enough has been freed, break to avoid holding the lock too long */ - if (total_freed_pages >= HEAP_RECLAIM_SCAN_BATCH_SIZE) + if (total_freed_pages >= max_pages) break; } @@ -313,7 +367,8 @@ static unsigned long kbase_csf_tiler_heap_reclaim_scan_free_pages(struct kbase_d avail = (unsigned long)atomic_read(&mgr->unused_pages); if (avail) - freed = reclaim_unused_heap_pages(kbdev); + freed = kbase_csf_tiler_heap_reclaim_unused_pages(kbdev, + HEAP_RECLAIM_SCENARIO_SHRINKER); mutex_unlock(&kbdev->csf.scheduler.lock); @@ -372,6 +427,9 @@ int kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev) prio++) INIT_LIST_HEAD(&scheduler->reclaim_mgr.ctx_lists[prio]); + scheduler->reclaim_mgr.offslot_setting.timeout_ms = HEAP_RECLAIM_OFFSLOT_TIMEOUT_MS; + scheduler->reclaim_mgr.offslot_setting.pages = HEAP_RECLAIM_OFFSLOT_PAGES; + reclaim->count_objects = kbase_csf_tiler_heap_reclaim_count_objects; reclaim->scan_objects = kbase_csf_tiler_heap_reclaim_scan_objects; reclaim->seeks = HEAP_SHRINKER_SEEKS; diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.h index d41b7baabd02..e74bd4f25926 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_tiler_heap_reclaim.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -79,4 +79,26 @@ int kbase_csf_tiler_heap_reclaim_mgr_init(struct kbase_device *kbdev); */ void kbase_csf_tiler_heap_reclaim_mgr_term(struct kbase_device *kbdev); +/** + * enum heap_reclaim_scenario - heap reclaim scenario list. + * + * @HEAP_RECLAIM_SCENARIO_SHRINKER: Reclaim through shrinker. + * @HEAP_RECLAIM_SCENARIO_SCHEDULER: Reclaim through scheduler. + * @HEAP_RECLAIM_SCENARIO_COUNT: The number of reclaim scenarios. + */ +enum heap_reclaim_scenario { + HEAP_RECLAIM_SCENARIO_SHRINKER = 0, + HEAP_RECLAIM_SCENARIO_SCHEDULER, + HEAP_RECLAIM_SCENARIO_COUNT +}; + +/** + * kbase_csf_tiler_heap_reclaim_unused_pages - Reclaim unused heap pages. + * @kbdev: Pointer to the device. + * @scenario: Reclaim scenario. + * + * Return: total freed pages. + */ +unsigned long kbase_csf_tiler_heap_reclaim_unused_pages(struct kbase_device *kbdev, + enum heap_reclaim_scenario scenario); #endif diff --git a/drivers/gpu/arm/valhall/debug/Kbuild b/drivers/gpu/arm/valhall/debug/Kbuild index a27458a51d4d..74b20f5ea3dc 100644 --- a/drivers/gpu/arm/valhall/debug/Kbuild +++ b/drivers/gpu/arm/valhall/debug/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2021-2022 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2021-2022, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -18,11 +18,7 @@ # # -valhall_kbase-y += debug/mali_kbase_debug_ktrace.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += debug/backend/mali_kbase_debug_ktrace_csf.o - valhall_kbase-$(CONFIG_MALI_VALHALL_CORESIGHT) += debug/backend/mali_kbase_debug_coresight_csf.o -else - valhall_kbase-y += debug/backend/mali_kbase_debug_ktrace_jm.o -endif +valhall_kbase-y += \ + debug/mali_kbase_debug_ktrace.o \ + debug/backend/mali_kbase_debug_ktrace_csf.o +valhall_kbase-$(CONFIG_MALI_VALHALL_CORESIGHT) += debug/backend/mali_kbase_debug_coresight_csf.o diff --git a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_codes_jm.h b/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_codes_jm.h deleted file mode 100644 index 14dcc265bc90..000000000000 --- a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_codes_jm.h +++ /dev/null @@ -1,203 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** - * ***** DO NOT INCLUDE DIRECTLY ***** - * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** - */ - -/* - * The purpose of this header file is just to contain a list of trace code - * identifiers - * - * When updating this file, also remember to update - * mali_kbase_debug_linux_ktrace_jm.h - * - * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT - * DESCRIBED IN mali_kbase_debug_ktrace_codes.h - */ - -#if 0 /* Dummy section to avoid breaking formatting */ -int dummy_array[] = { -#endif - -/* - * Job Slot management events - */ -/* info_val==irq rawstat at start */ -KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ), - /* info_val==jobs processed */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ_END), - /* In the following: - * - * - ctx is set if a corresponding job found (NULL otherwise, e.g. some - * soft-stop cases) - * - uatom==kernel-side mapped uatom address (for correlation with - * user-side) - */ - /* info_val==exit code; gpu_addr==chain gpuaddr */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_JOB_DONE), - /* gpu_addr==JS_HEAD read - * info_val==event code - */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_RETURN_ATOM_TO_JS), - /* gpu_addr==JS_HEAD read - * info_val==event code - */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_MARK_FOR_RETURN_TO_JS), - /* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of - * affinity - */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT), - /* gpu_addr is as follows: - * - If JS_STATUS active after soft-stop, val==gpu addr written to - * JS_HEAD on submit - * - otherwise gpu_addr==0 - */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP), KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0), - KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1), - /* gpu_addr==JS_HEAD read */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP), - /* gpu_addr==JS_HEAD read */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_0), - /* gpu_addr==JS_HEAD read */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_1), - /* gpu_addr==JS_TAIL read */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD), - /* gpu_addr is as follows: - * - If JS_STATUS active before soft-stop, val==JS_HEAD - * - otherwise gpu_addr==0 - */ - /* gpu_addr==JS_HEAD read */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_CHECK_HEAD), KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS), - KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE), - /* info_val == is_scheduled */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED), - /* info_val == is_scheduled */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED), KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_DONE), - /* info_val == nr jobs submitted */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP), - /* gpu_addr==JS_HEAD_NEXT last written */ - KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_EVICT), - KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET), - KBASE_KTRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER), - KBASE_KTRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER), - /* - * Job dispatch events - */ - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB), - /* gpu_addr==0, info_val==0, uatom==0 */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER), - /* - * Scheduler Core events - */ - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_ADD_JOB), - /* gpu_addr==last value written/would be written to JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_REMOVE_JOB), - /* gpu_addr==value to write into JS_HEAD - * info_val==priority of atom as a KBASE_JS_ATOM_SCHED_PRIO_<...> value - * (0 highest) - */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_PULL_JOB), - /* gpu_addr==value that would be written to JS_HEAD if run again */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_UNPULL_JOB), - KBASE_KTRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED), - KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED), - /* info_val == lower 32 bits of affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT), - /* info_val == lower 32 bits of affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED), - /* info_val == lower 32 bits of affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED), - /* info_val == lower 32 bits of rechecked affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED), - /* info_val == lower 32 bits of rechecked affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED), - /* info_val == lower 32 bits of affinity */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE), - /* info_val == the ctx attribute now on ctx */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX), - /* info_val == the ctx attribute now on runpool */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL), - /* info_val == the ctx attribute now off ctx */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX), - /* info_val == the ctx attribute now off runpool */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_RETURN_WORKER), - /* gpu_addr==value to write into JS_HEAD */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_RETURN_WORKER_END), - /* info_val==priority level blocked (0 highest) */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_SLOT_PRIO_BLOCKED), - /* info_val==priority level unblocked (0 highest) - * note that the priority level may still be blocked on higher levels - */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_SLOT_PRIO_UNBLOCKED), - /* gpu_addr==value to write into JS_HEAD - * info_val==priority level unblocked - priorities at this and higher - * are unblocked (0 highest) - */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_SLOT_PRIO_AND_HIGHER_UNBLOCKED), - /* gpu_addr==value to write into JS_HEAD - * info_val==priority level blocked (0 highest) - */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_SLOT_PRIO_IS_BLOCKED), - /* - * Scheduler Policy events - */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX), - /* info_val == whether it was evicted */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ), - /* gpu_addr==JS_HEAD to write if the job were run */ - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START), - KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END), - -#if 0 /* Dummy section to avoid breaking formatting */ -}; -#endif - - /* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ diff --git a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_defs_jm.h b/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_defs_jm.h deleted file mode 100644 index 9978e4939d14..000000000000 --- a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_defs_jm.h +++ /dev/null @@ -1,111 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_DEBUG_KTRACE_DEFS_JM_H_ -#define _KBASE_DEBUG_KTRACE_DEFS_JM_H_ - -#if KBASE_KTRACE_TARGET_RBUF -/** - * DOC: KTrace version history, JM variant - * - * 1.0: - * Original version (implicit, header did not carry version information). - * - * 2.0: - * Introduced version information into the header. - * - * Some changes of parameter names in header. - * - * Trace now uses all 64-bits of info_val. - * - * Non-JM specific parts moved to using info_val instead of refcount/gpu_addr. - * - * 2.1: - * kctx field is no longer a pointer, and is now an ID of the format %d_%u as - * used by kctx directories in mali debugfs entries: (tgid creating the kctx), - * (unique kctx id). - * - * ftrace backend now outputs kctx field (as %d_%u format). - * - * 2.2: - * Add tracing codes for pulling, unpulling, and returns atoms to JS for - * diagnosing soft-stop path and preemption problems - */ -#define KBASE_KTRACE_VERSION_MAJOR 2 -#define KBASE_KTRACE_VERSION_MINOR 2 -#endif /* KBASE_KTRACE_TARGET_RBUF */ - -/* - * Note: mali_kbase_debug_ktrace_jm.h needs these value even if the RBUF target - * is disabled (they get discarded with CSTD_UNUSED(), but they're still - * referenced) - */ - -/* indicates if the trace message has a valid refcount member */ -#define KBASE_KTRACE_FLAG_JM_REFCOUNT (((kbase_ktrace_flag_t)1) << 0) -/* indicates if the trace message has a valid jobslot member */ -#define KBASE_KTRACE_FLAG_JM_JOBSLOT (((kbase_ktrace_flag_t)1) << 1) -/* indicates if the trace message has valid atom related info. */ -#define KBASE_KTRACE_FLAG_JM_ATOM (((kbase_ktrace_flag_t)1) << 2) - -#if KBASE_KTRACE_TARGET_RBUF -/* Collect all the flags together for debug checking */ -#define KBASE_KTRACE_FLAG_BACKEND_ALL \ - (KBASE_KTRACE_FLAG_JM_REFCOUNT | KBASE_KTRACE_FLAG_JM_JOBSLOT | KBASE_KTRACE_FLAG_JM_ATOM) - -/** - * union kbase_ktrace_backend - backend specific part of a trace message - * Contains only a struct but is a union such that it is compatible with - * generic JM and CSF KTrace calls. - * - * @gpu: gpu union member - * @gpu.atom_udata: Copy of the user data sent for the atom in base_jd_submit. - * Only valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags - * @gpu.gpu_addr: GPU address, usually of the job-chain represented by an - * atom. - * @gpu.atom_number: id of the atom for which trace message was added. Only - * valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags - * @gpu.code: Identifies the event, refer to enum kbase_ktrace_code. - * @gpu.flags: indicates information about the trace message itself. Used - * during dumping of the message. - * @gpu.jobslot: job-slot for which trace message was added, valid only for - * job-slot management events. - * @gpu.refcount: reference count for the context, valid for certain events - * related to scheduler core and policy. - */ -union kbase_ktrace_backend { - struct { - /* Place 64 and 32-bit members together */ - u64 atom_udata[2]; /* Only valid for - * KBASE_KTRACE_FLAG_JM_ATOM - */ - u64 gpu_addr; - unsigned int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ - /* Pack smaller members together */ - kbase_ktrace_code_t code; - kbase_ktrace_flag_t flags; - u8 jobslot; - u8 refcount; - } gpu; -}; -#endif /* KBASE_KTRACE_TARGET_RBUF */ - -#endif /* _KBASE_DEBUG_KTRACE_DEFS_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.c b/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.c deleted file mode 100644 index 39306e7d45e3..000000000000 --- a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.c +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include "debug/mali_kbase_debug_ktrace_internal.h" -#include "debug/backend/mali_kbase_debug_ktrace_jm.h" - -#if KBASE_KTRACE_TARGET_RBUF - -void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) -{ - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - "katom,gpu_addr,jobslot,refcount"), - 0); -} - -void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char *buffer, int sz, - s32 *written) -{ - /* katom */ - if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_ATOM) - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - "atom %u (ud: 0x%llx 0x%llx)", - trace_msg->backend.gpu.atom_number, - trace_msg->backend.gpu.atom_udata[0], - trace_msg->backend.gpu.atom_udata[1]), - 0); - - /* gpu_addr */ - if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_BACKEND) - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), - ",%.8llx,", trace_msg->backend.gpu.gpu_addr), - 0); - else - *written += - MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,"), 0); - - /* jobslot */ - if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", - trace_msg->backend.gpu.jobslot), - 0); - - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); - - /* refcount */ - if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) - *written += MAX(scnprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", - trace_msg->backend.gpu.refcount), - 0); -} - -void kbasep_ktrace_add_jm(struct kbase_device *kbdev, enum kbase_ktrace_code code, - struct kbase_context *kctx, const struct kbase_jd_atom *katom, - u64 gpu_addr, kbase_ktrace_flag_t flags, int refcount, - unsigned int jobslot, u64 info_val) -{ - unsigned long irqflags; - struct kbase_ktrace_msg *trace_msg; - - if (unlikely(!kbasep_ktrace_initialized(&kbdev->ktrace))) - return; - - spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); - - /* Reserve and update indices */ - trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); - - /* Fill the common part of the message (including backend.gpu.flags) */ - kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, info_val); - - /* Indicate to the common code that backend-specific parts will be - * valid - */ - trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_BACKEND; - - /* Fill the JM-specific parts of the message */ - if (katom) { - trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_JM_ATOM; - - trace_msg->backend.gpu.atom_number = kbase_jd_atom_id(katom->kctx, katom); - trace_msg->backend.gpu.atom_udata[0] = katom->udata.blob[0]; - trace_msg->backend.gpu.atom_udata[1] = katom->udata.blob[1]; - } - - trace_msg->backend.gpu.gpu_addr = gpu_addr; - trace_msg->backend.gpu.jobslot = jobslot; - /* Clamp refcount */ - trace_msg->backend.gpu.refcount = MIN((unsigned int)refcount, 0xFF); - - WARN_ON((trace_msg->backend.gpu.flags & ~KBASE_KTRACE_FLAG_ALL)); - - /* Done */ - spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); -} - -#endif /* KBASE_KTRACE_TARGET_RBUF */ diff --git a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.h b/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.h deleted file mode 100644 index 59ad23d7677c..000000000000 --- a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_ktrace_jm.h +++ /dev/null @@ -1,297 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_DEBUG_KTRACE_JM_H_ -#define _KBASE_DEBUG_KTRACE_JM_H_ - -/* - * KTrace target for internal ringbuffer - */ -#if KBASE_KTRACE_TARGET_RBUF -/** - * kbasep_ktrace_add_jm - internal function to add trace about Job Management - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @flags: flags about the message - * @refcount: reference count information to add to the trace - * @jobslot: jobslot information to add to the trace - * @info_val: generic information about @code to add to the trace - * - * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_JM() instead. - */ -void kbasep_ktrace_add_jm(struct kbase_device *kbdev, enum kbase_ktrace_code code, - struct kbase_context *kctx, const struct kbase_jd_atom *katom, - u64 gpu_addr, kbase_ktrace_flag_t flags, int refcount, - unsigned int jobslot, u64 info_val); - -#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, refcount, jobslot, \ - info_val) \ - kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, gpu_addr, flags, \ - refcount, jobslot, info_val) - -#else /* KBASE_KTRACE_TARGET_RBUF */ - -#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, refcount, jobslot, \ - info_val) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(flags); \ - CSTD_UNUSED(refcount); \ - CSTD_UNUSED(jobslot); \ - CSTD_UNUSED(info_val); \ - CSTD_NOP(0); \ - } while (0) -#endif /* KBASE_KTRACE_TARGET_RBUF */ - -/* - * KTrace target for Linux's ftrace - * - * Note: the header file(s) that define the trace_mali_<...> tracepoints are - * included by the parent header file - */ -#if KBASE_KTRACE_TARGET_FTRACE -#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, jobslot) \ - trace_mali_##code(kctx, jobslot, 0) - -#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, jobslot, \ - info_val) \ - trace_mali_##code(kctx, jobslot, info_val) - -#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, refcount) \ - trace_mali_##code(kctx, refcount, 0) - -#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, gpu_addr, refcount, \ - info_val) \ - trace_mali_##code(kctx, refcount, info_val) - -#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ - trace_mali_##code(kctx, gpu_addr, info_val) -#else /* KBASE_KTRACE_TARGET_FTRACE */ -#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, jobslot) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(jobslot); \ - CSTD_NOP(0); \ - } while (0) - -#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, jobslot, \ - info_val) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(jobslot); \ - CSTD_UNUSED(info_val); \ - CSTD_NOP(0); \ - } while (0) - -#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, refcount) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(refcount); \ - CSTD_NOP(0); \ - } while (0) - -#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, gpu_addr, refcount, \ - info_val) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(info_val); \ - CSTD_NOP(0); \ - } while (0) - -#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ - do { \ - CSTD_UNUSED(kbdev); \ - CSTD_NOP(code); \ - CSTD_UNUSED(kctx); \ - CSTD_UNUSED(katom); \ - CSTD_UNUSED(gpu_addr); \ - CSTD_UNUSED(info_val); \ - CSTD_NOP(0); \ - } while (0) -#endif /* KBASE_KTRACE_TARGET_FTRACE */ - -/* - * Master set of macros to route KTrace to any of the targets - */ - -/** - * KBASE_KTRACE_ADD_JM_SLOT - Add trace values about a job-slot - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @jobslot: jobslot information to add to the trace - * - * Note: Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_VALHALL_DEBUG not defined). Therefore, when - * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied - * to this macro must: - * a) be static or static inline, and - * b) just return 0 and have no other statements present in the body. - */ -#define KBASE_KTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, jobslot) \ - do { \ - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - unsigned int __jobslot = jobslot; \ - KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ - KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, 0); \ - KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \ - } while (0) - -/** - * KBASE_KTRACE_ADD_JM_SLOT_INFO - Add trace values about a job-slot, with info - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @jobslot: jobslot information to add to the trace - * @info_val: generic information about @code to add to the trace - * - * Note: Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_VALHALL_DEBUG not defined). Therefore, when - * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied - * to this macro must: - * a) be static or static inline, and - * b) just return 0 and have no other statements present in the body. - */ -#define KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, jobslot, info_val) \ - do { \ - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - unsigned int __jobslot = jobslot; \ - u64 __info_val = info_val; \ - KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ - KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, __info_val); \ - KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, \ - __jobslot, __info_val); \ - } while (0) - -/** - * KBASE_KTRACE_ADD_JM_REFCOUNT - Add trace values about a kctx refcount - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @refcount: reference count information to add to the trace - * - * Note: Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_VALHALL_DEBUG not defined). Therefore, when - * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied - * to this macro must: - * a) be static or static inline, and - * b) just return 0 and have no other statements present in the body. - */ -#define KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, refcount) \ - do { \ - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - int __refcount = refcount; \ - KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ - KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, 0u); \ - KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, \ - (unsigned int)__refcount); \ - } while (0) - -/** - * KBASE_KTRACE_ADD_JM_REFCOUNT_INFO - Add trace values about a kctx refcount, - * and info - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @refcount: reference count information to add to the trace - * @info_val: generic information about @code to add to the trace - * - * Note: Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_VALHALL_DEBUG not defined). Therefore, when - * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied - * to this macro must: - * a) be static or static inline, and - * b) just return 0 and have no other statements present in the body. - */ -#define KBASE_KTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, gpu_addr, refcount, info_val) \ - do { \ - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - int __refcount = refcount; \ - u64 __info_val = info_val; \ - KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ - KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, \ - __info_val); \ - KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, \ - (unsigned int)__refcount, __info_val); \ - } while (0) - -/** - * KBASE_KTRACE_ADD_JM - Add trace values (no slot or refcount) - * @kbdev: kbase device - * @code: trace code - * @kctx: kbase context, or NULL if no context - * @katom: kbase atom, or NULL if no atom - * @gpu_addr: GPU address, usually related to @katom - * @info_val: generic information about @code to add to the trace - * - * Note: Any functions called through this macro will still be evaluated in - * Release builds (CONFIG_MALI_VALHALL_DEBUG not defined). Therefore, when - * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied - * to this macro must: - * a) be static or static inline, and - * b) just return 0 and have no other statements present in the body. - */ -#define KBASE_KTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ - do { \ - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - u64 __info_val = info_val; \ - KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, 0u, 0, 0, \ - __info_val); \ - KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \ - } while (0) - -#endif /* _KBASE_DEBUG_KTRACE_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_linux_ktrace_jm.h b/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_linux_ktrace_jm.h deleted file mode 100644 index 3e5b0ec89758..000000000000 --- a/drivers/gpu/arm/valhall/debug/backend/mali_kbase_debug_linux_ktrace_jm.h +++ /dev/null @@ -1,151 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * NOTE: This must **only** be included through mali_linux_trace.h, - * otherwise it will fail to setup tracepoints correctly - */ - -#if !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ) -#define _KBASE_DEBUG_LINUX_KTRACE_JM_H_ - -DECLARE_EVENT_CLASS(mali_jm_slot_template, - TP_PROTO(struct kbase_context *kctx, unsigned int jobslot, u64 info_val), - TP_ARGS(kctx, jobslot, info_val), - TP_STRUCT__entry(__field(pid_t, kctx_tgid) __field(u32, kctx_id) - __field(unsigned int, jobslot) __field(u64, info_val)), - TP_fast_assign(__entry->kctx_id = (kctx) ? kctx->id : 0u; - __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; - __entry->jobslot = jobslot; __entry->info_val = info_val;), - TP_printk("kctx=%d_%u jobslot=%u info=0x%llx", __entry->kctx_tgid, - __entry->kctx_id, __entry->jobslot, __entry->info_val)); - -#define DEFINE_MALI_JM_SLOT_EVENT(name) \ - DEFINE_EVENT(mali_jm_slot_template, mali_##name, \ - TP_PROTO(struct kbase_context *kctx, unsigned int jobslot, u64 info_val), \ - TP_ARGS(kctx, jobslot, info_val)) -DEFINE_MALI_JM_SLOT_EVENT(JM_RETURN_ATOM_TO_JS); -DEFINE_MALI_JM_SLOT_EVENT(JM_MARK_FOR_RETURN_TO_JS); -DEFINE_MALI_JM_SLOT_EVENT(JM_SUBMIT); -DEFINE_MALI_JM_SLOT_EVENT(JM_JOB_DONE); -DEFINE_MALI_JM_SLOT_EVENT(JM_UPDATE_HEAD); -DEFINE_MALI_JM_SLOT_EVENT(JM_CHECK_HEAD); -DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP); -DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_0); -DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_1); -DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP); -DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_0); -DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_1); -DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP); -DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_EVICT); -DEFINE_MALI_JM_SLOT_EVENT(JM_BEGIN_RESET_WORKER); -DEFINE_MALI_JM_SLOT_EVENT(JM_END_RESET_WORKER); -DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED); -DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED); -DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_CURRENT); -DEFINE_MALI_JM_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB); -DEFINE_MALI_JM_SLOT_EVENT(JS_PULL_JOB); -DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED); -DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED); -DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED); -DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE); -DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB); -DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED); -DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB); -DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); -DEFINE_MALI_JM_SLOT_EVENT(JS_SLOT_PRIO_BLOCKED); -DEFINE_MALI_JM_SLOT_EVENT(JS_SLOT_PRIO_UNBLOCKED); -DEFINE_MALI_JM_SLOT_EVENT(JS_SLOT_PRIO_AND_HIGHER_UNBLOCKED); -DEFINE_MALI_JM_SLOT_EVENT(JS_SLOT_PRIO_IS_BLOCKED); -#undef DEFINE_MALI_JM_SLOT_EVENT - -DECLARE_EVENT_CLASS(mali_jm_refcount_template, - TP_PROTO(struct kbase_context *kctx, unsigned int refcount, u64 info_val), - TP_ARGS(kctx, refcount, info_val), - TP_STRUCT__entry(__field(pid_t, kctx_tgid) __field(u32, kctx_id) - __field(unsigned int, refcount) - __field(u64, info_val)), - TP_fast_assign(__entry->kctx_id = (kctx) ? kctx->id : 0u; - __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; - __entry->refcount = refcount; __entry->info_val = info_val;), - TP_printk("kctx=%d_%u refcount=%u info=0x%llx", __entry->kctx_tgid, - __entry->kctx_id, __entry->refcount, __entry->info_val)); - -#define DEFINE_MALI_JM_REFCOUNT_EVENT(name) \ - DEFINE_EVENT(mali_jm_refcount_template, mali_##name, \ - TP_PROTO(struct kbase_context *kctx, unsigned int refcount, u64 info_val), \ - TP_ARGS(kctx, refcount, info_val)) -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_ADD_JOB); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_REMOVE_JOB); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_INIT_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TERM_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX); -DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); -#undef DEFINE_MALI_JM_REFCOUNT_EVENT - -DECLARE_EVENT_CLASS(mali_jm_add_template, - TP_PROTO(struct kbase_context *kctx, u64 gpu_addr, u64 info_val), - TP_ARGS(kctx, gpu_addr, info_val), - TP_STRUCT__entry(__field(pid_t, kctx_tgid) __field(u32, kctx_id) - __field(u64, gpu_addr) __field(u64, info_val)), - TP_fast_assign(__entry->kctx_id = (kctx) ? kctx->id : 0u; - __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; - __entry->gpu_addr = gpu_addr; __entry->info_val = info_val;), - TP_printk("kctx=%d_%u gpu_addr=0x%llx info=0x%llx", __entry->kctx_tgid, - __entry->kctx_id, __entry->gpu_addr, __entry->info_val)); - -#define DEFINE_MALI_JM_ADD_EVENT(name) \ - DEFINE_EVENT(mali_jm_add_template, mali_##name, \ - TP_PROTO(struct kbase_context *kctx, u64 gpu_addr, u64 info_val), \ - TP_ARGS(kctx, gpu_addr, info_val)) -DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER); -DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER_END); -DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL_WORKER); -DEFINE_MALI_JM_ADD_EVENT(JD_DONE); -DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL); -DEFINE_MALI_JM_ADD_EVENT(JD_ZAP_CONTEXT); -DEFINE_MALI_JM_ADD_EVENT(JM_IRQ); -DEFINE_MALI_JM_ADD_EVENT(JM_IRQ_END); -DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS); -DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS_DONE); -DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_NON_SCHEDULED); -DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_SCHEDULED); -DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_DONE); -DEFINE_MALI_JM_ADD_EVENT(JM_SUBMIT_AFTER_RESET); -DEFINE_MALI_JM_ADD_EVENT(JM_JOB_COMPLETE); -DEFINE_MALI_JM_ADD_EVENT(JS_UNPULL_JOB); -DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL); -DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL); -DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX); -DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX); -DEFINE_MALI_JM_ADD_EVENT(JS_RETURN_WORKER); -DEFINE_MALI_JM_ADD_EVENT(JS_RETURN_WORKER_END); -DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_END); -DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_START); -DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_ENQUEUE_JOB); -#undef DEFINE_MALI_JM_ADD_EVENT - -#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ)*/ diff --git a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace.h b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace.h index 9d12a000c5bb..53c1debc840c 100644 --- a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace.h +++ b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -40,11 +40,7 @@ #include "mali_linux_trace.h" #endif -#if MALI_USE_CSF #include "debug/backend/mali_kbase_debug_ktrace_csf.h" -#else -#include "debug/backend/mali_kbase_debug_ktrace_jm.h" -#endif /** * kbase_ktrace_init - initialize kbase ktrace. diff --git a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_codes.h b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_codes.h index f5810d477c83..94c5ae0dd290 100644 --- a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_codes.h +++ b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_codes.h @@ -165,11 +165,7 @@ KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STOPPED), KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_REQUESTED), -#if MALI_USE_CSF #include "debug/backend/mali_kbase_debug_ktrace_codes_csf.h" -#else -#include "debug/backend/mali_kbase_debug_ktrace_codes_jm.h" -#endif /* * Unused code just to make it easier to not have a comma at the end. * All other codes MUST come before this diff --git a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_defs.h b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_defs.h index be91473b81ec..9907c74efcc0 100644 --- a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_defs.h +++ b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_ktrace_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -91,11 +91,7 @@ union kbase_ktrace_backend; #endif /* KBASE_KTRACE_TARGET_RBUF */ -#if MALI_USE_CSF #include "debug/backend/mali_kbase_debug_ktrace_defs_csf.h" -#else -#include "debug/backend/mali_kbase_debug_ktrace_defs_jm.h" -#endif #if KBASE_KTRACE_TARGET_RBUF /* Indicates if the trace message has backend related info. diff --git a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_linux_ktrace.h b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_linux_ktrace.h index 76ac6a6c429b..224a758d2c2c 100644 --- a/drivers/gpu/arm/valhall/debug/mali_kbase_debug_linux_ktrace.h +++ b/drivers/gpu/arm/valhall/debug/mali_kbase_debug_linux_ktrace.h @@ -111,11 +111,7 @@ DEFINE_MALI_ADD_EVENT(ARB_GPU_STOP_REQUESTED); DEFINE_MALI_ADD_EVENT(ARB_GPU_STOPPED); DEFINE_MALI_ADD_EVENT(ARB_GPU_REQUESTED); -#if MALI_USE_CSF #include "backend/mali_kbase_debug_linux_ktrace_csf.h" -#else -#include "backend/mali_kbase_debug_linux_ktrace_jm.h" -#endif #undef DEFINE_MALI_ADD_EVENT diff --git a/drivers/gpu/arm/valhall/device/Kbuild b/drivers/gpu/arm/valhall/device/Kbuild index 7b4057023574..5a50d974c080 100644 --- a/drivers/gpu/arm/valhall/device/Kbuild +++ b/drivers/gpu/arm/valhall/device/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2021, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -20,14 +20,6 @@ valhall_kbase-y += \ device/mali_kbase_device.o \ - device/mali_kbase_device_hw.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += \ - device/backend/mali_kbase_device_csf.o \ - device/backend/mali_kbase_device_hw_csf.o -else - valhall_kbase-y += \ - device/backend/mali_kbase_device_jm.o \ - device/backend/mali_kbase_device_hw_jm.o -endif + device/mali_kbase_device_hw.o \ + device/backend/mali_kbase_device_csf.o \ + device/backend/mali_kbase_device_hw_csf.o diff --git a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_csf.c b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_csf.c index 4362dd1de561..539590737d90 100644 --- a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_csf.c +++ b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_csf.c @@ -61,6 +61,7 @@ static void kbase_device_firmware_hwcnt_term(struct kbase_device *kbdev) kbase_kinstr_prfcnt_term(kbdev->kinstr_prfcnt_ctx); kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt); kbase_hwcnt_backend_csf_metadata_term(&kbdev->hwcnt_gpu_iface); + kbase_hwcnt_backend_csf_ring_buf_term(&kbdev->hwcnt_gpu_iface); kbase_csf_firmware_unload_term(kbdev); } } diff --git a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_csf.c b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_csf.c index 792d2efa7706..e9019425f688 100644 --- a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_csf.c +++ b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_csf.c @@ -82,12 +82,10 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; bool is_legacy_gpu_irq_mask = true; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { power_changed_mask = MCU_STATUS_GPU_IRQ; is_legacy_gpu_irq_mask = false; } -#endif KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); @@ -136,15 +134,10 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) val &= ~GPU_PROTECTED_FAULT; } -#if MALI_USE_CSF if (!kbdev->pm.backend.has_host_pwr_iface) { if (val & RESET_COMPLETED) kbase_pm_reset_done(kbdev); } -#else - if (val & RESET_COMPLETED) - kbase_pm_reset_done(kbdev); -#endif /* Defer clearing CLEAN_CACHES_COMPLETED to kbase_clean_caches_done. * We need to acquire hwaccess_lock to avoid a race condition with @@ -153,8 +146,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val & ~CLEAN_CACHES_COMPLETED); kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_CLEAR), val & ~CLEAN_CACHES_COMPLETED); -#ifdef KBASE_PM_RUNTIME - if (val & DOORBELL_MIRROR) { + if (IS_ENABLED(CONFIG_PM) && (val & DOORBELL_MIRROR)) { unsigned long flags; dev_dbg(kbdev->dev, "Doorbell mirror interrupt received"); @@ -213,7 +205,6 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -#endif /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must * be called after the IRQ has been cleared. This is because it might diff --git a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_jm.c b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_jm.c deleted file mode 100644 index 368ffe257771..000000000000 --- a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_hw_jm.c +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -/** - * kbase_report_gpu_fault - Report a GPU fault. - * @kbdev: Kbase device pointer - * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS - * was also set - * - * This function is called from the interrupt handler when a GPU fault occurs. - * It reports the details of the fault using dev_warn(). - */ -static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) -{ - u32 status = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_FAULTSTATUS)); - uintptr_t phys_addr = kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(GPU_FAULTADDRESS)); - - dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at PA 0x%pK", status, - kbase_gpu_exception_name(status & 0xFF), (void *)phys_addr); - if (multiple) - dev_warn(kbdev->dev, - "There were multiple GPU faults - some have not been reported\n"); - -} - -void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) -{ - KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); - if (val & GPU_FAULT) - kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS); - - if (val & RESET_COMPLETED) - kbase_pm_reset_done(kbdev); - - /* Defer clearing CLEAN_CACHES_COMPLETED to kbase_clean_caches_done. - * We need to acquire hwaccess_lock to avoid a race condition with - * kbase_gpu_cache_flush_and_busy_wait - */ - KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val & ~CLEAN_CACHES_COMPLETED); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_CLEAR), val & ~CLEAN_CACHES_COMPLETED); - - /* kbase_instr_hwcnt_sample_done frees the HWCNT pipeline to request another - * sample. Therefore this must be called after clearing the IRQ to avoid a - * race between clearing and the next sample raising the IRQ again. - */ - if (val & PRFCNT_SAMPLE_COMPLETED) - kbase_instr_hwcnt_sample_done(kbdev); - - /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must - * be called after the IRQ has been cleared. This is because it might - * trigger further power transitions and we don't want to miss the - * interrupt raised to notify us that these further transitions have - * finished. The same applies to kbase_clean_caches_done() - if another - * clean was queued, it might trigger another clean, which might - * generate another interrupt which shouldn't be missed. - */ - - if (val & CLEAN_CACHES_COMPLETED) - kbase_clean_caches_done(kbdev); - - if (val & POWER_CHANGED_ALL) { - kbase_pm_power_changed(kbdev); - } else if (val & CLEAN_CACHES_COMPLETED) { - /* If cache line evict messages can be lost when shader cores - * power down then we need to flush the L2 cache before powering - * down cores. When the flush completes, the shaders' state - * machine needs to be re-invoked to proceed with powering down - * cores. - */ - if (kbdev->pm.backend.l2_always_on || - kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) - kbase_pm_power_changed(kbdev); - } - - KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); -} -KBASE_EXPORT_TEST_API(kbase_gpu_interrupt); diff --git a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_jm.c b/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_jm.c deleted file mode 100644 index 0c96b2ba3eba..000000000000 --- a/drivers/gpu/arm/valhall/device/backend/mali_kbase_device_jm.c +++ /dev/null @@ -1,344 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) -#include -#endif - -/** - * kbase_backend_late_init - Perform any backend-specific initialization. - * @kbdev: Device pointer - * - * Return: 0 on success, or an error code on failure. - */ -static int kbase_backend_late_init(struct kbase_device *kbdev) -{ - int err; - - err = kbase_hwaccess_pm_init(kbdev); - if (err) - return err; - - err = kbase_reset_gpu_init(kbdev); - if (err) - goto fail_reset_gpu_init; - - err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); - if (err) - goto fail_pm_powerup; - - err = kbase_backend_timer_init(kbdev); - if (err) - goto fail_timer; - -#ifdef CONFIG_MALI_VALHALL_DEBUG -#if IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) - if (kbase_validate_interrupts(kbdev) != 0) { - dev_err(kbdev->dev, "Interrupt validation failed.\n"); - err = -EINVAL; - goto fail_interrupt_test; - } -#endif /* IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) */ -#endif /* CONFIG_MALI_VALHALL_DEBUG */ - - err = kbase_job_slot_init(kbdev); - if (err) - goto fail_job_slot; - - /* Do the initialisation of devfreq. - * Devfreq needs backend_timer_init() for completion of its - * initialisation and it also needs to catch the first callback - * occurrence of the runtime_suspend event for maintaining state - * coherence with the backend power management, hence needs to be - * placed before the kbase_pm_context_idle(). - */ - err = kbase_backend_devfreq_init(kbdev); - if (err) - goto fail_devfreq_init; - - /* Update gpuprops with L2_FEATURES if applicable */ - err = kbase_gpuprops_update_l2_features(kbdev); - if (err) - goto fail_update_l2_features; - - init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); - - /* Idle the GPU and/or cores, if the policy wants it to */ - kbase_pm_context_idle(kbdev); - - mutex_init(&kbdev->fw_load_lock); - - return 0; - -fail_update_l2_features: - kbase_backend_devfreq_term(kbdev); -fail_devfreq_init: - kbase_job_slot_term(kbdev); -fail_job_slot: - -#ifdef CONFIG_MALI_VALHALL_DEBUG -#if IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) -fail_interrupt_test: -#endif /* IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) */ -#endif /* CONFIG_MALI_VALHALL_DEBUG */ - - kbase_backend_timer_term(kbdev); -fail_timer: - kbase_pm_context_idle(kbdev); - kbase_hwaccess_pm_halt(kbdev); -fail_pm_powerup: - kbase_reset_gpu_term(kbdev); -fail_reset_gpu_init: - kbase_hwaccess_pm_term(kbdev); - - return err; -} - -/** - * kbase_backend_late_term - Perform any backend-specific termination. - * @kbdev: Device pointer - */ -static void kbase_backend_late_term(struct kbase_device *kbdev) -{ - kbase_backend_devfreq_term(kbdev); - kbase_job_slot_halt(kbdev); - kbase_job_slot_term(kbdev); - kbase_backend_timer_term(kbdev); - kbase_hwaccess_pm_halt(kbdev); - kbase_reset_gpu_term(kbdev); - kbase_hwaccess_pm_term(kbdev); -} - -/** - * kbase_device_hwcnt_watchdog_if_init - Create hardware counter watchdog - * interface. - * @kbdev: Device pointer - * Return: 0 on success, or an error code on failure. - */ -static int kbase_device_hwcnt_watchdog_if_init(struct kbase_device *kbdev) -{ - return kbase_hwcnt_watchdog_if_timer_create(&kbdev->hwcnt_watchdog_timer); -} - -/** - * kbase_device_hwcnt_watchdog_if_term - Terminate hardware counter watchdog - * interface. - * @kbdev: Device pointer - */ -static void kbase_device_hwcnt_watchdog_if_term(struct kbase_device *kbdev) -{ - kbase_hwcnt_watchdog_if_timer_destroy(&kbdev->hwcnt_watchdog_timer); -} - -/** - * kbase_device_hwcnt_backend_jm_init - Create hardware counter backend. - * @kbdev: Device pointer - * Return: 0 on success, or an error code on failure. - */ -static int kbase_device_hwcnt_backend_jm_init(struct kbase_device *kbdev) -{ - return kbase_hwcnt_backend_jm_create(kbdev, &kbdev->hwcnt_gpu_jm_backend); -} - -/** - * kbase_device_hwcnt_backend_jm_term - Terminate hardware counter backend. - * @kbdev: Device pointer - */ -static void kbase_device_hwcnt_backend_jm_term(struct kbase_device *kbdev) -{ - kbase_hwcnt_backend_jm_destroy(&kbdev->hwcnt_gpu_jm_backend); -} - -/** - * kbase_device_hwcnt_backend_jm_watchdog_init - Create hardware counter watchdog backend. - * @kbdev: Device pointer - * Return: 0 on success, or an error code on failure. - */ -static int kbase_device_hwcnt_backend_jm_watchdog_init(struct kbase_device *kbdev) -{ - const u32 timer_interval = - (kbdev->gpu_props.impl_tech == THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_FPGA) || - (kbdev->gpu_props.impl_tech == - THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_SOFTWARE) ? - HWCNT_BACKEND_WATCHDOG_TIMER_INTERVAL_FPGA_MS : - HWCNT_BACKEND_WATCHDOG_TIMER_INTERVAL_MS; - return kbase_hwcnt_backend_jm_watchdog_create(&kbdev->hwcnt_gpu_jm_backend, - &kbdev->hwcnt_watchdog_timer, - &kbdev->hwcnt_gpu_iface, timer_interval); -} - -/** - * kbase_device_hwcnt_backend_jm_watchdog_term - Terminate hardware counter watchdog backend. - * @kbdev: Device pointer - */ -static void kbase_device_hwcnt_backend_jm_watchdog_term(struct kbase_device *kbdev) -{ - kbase_hwcnt_backend_jm_watchdog_destroy(&kbdev->hwcnt_gpu_iface); -} - -static const struct kbase_device_init dev_init[] = { -#if !IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) - { kbase_gpu_device_create, kbase_gpu_device_destroy, "Dummy model initialization failed" }, -#else /* !IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) */ - { kbase_get_irqs, NULL, "IRQ search failed" }, - { registers_map, registers_unmap, "Register map failed" }, -#endif /* !IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) */ -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - { kbase_gpu_metrics_init, kbase_gpu_metrics_term, "GPU metrics initialization failed" }, -#endif /* IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) */ - { power_control_init, power_control_term, "Power control initialization failed" }, - { kbase_io_init, kbase_io_term, "Kbase IO initialization failed" }, - { kbase_device_io_history_init, kbase_device_io_history_term, - "Register access history initialization failed" }, - { kbase_device_early_init, kbase_device_early_term, "Early device initialization failed" }, - { kbase_backend_time_init, NULL, "Time backend initialization failed" }, - { kbase_device_misc_init, kbase_device_misc_term, - "Miscellaneous device initialization failed" }, - { kbase_device_pcm_dev_init, kbase_device_pcm_dev_term, - "Priority control manager initialization failed" }, - { kbase_ctx_sched_init, kbase_ctx_sched_term, "Context scheduler initialization failed" }, - { kbase_mem_init, kbase_mem_term, "Memory subsystem initialization failed" }, - { kbase_device_coherency_init, NULL, "Device coherency init failed" }, - { kbase_protected_mode_init, kbase_protected_mode_term, - "Protected mode subsystem initialization failed" }, - { kbase_device_list_init, kbase_device_list_term, "Device list setup failed" }, - { kbasep_js_devdata_init, kbasep_js_devdata_term, "Job JS devdata initialization failed" }, - { kbase_device_timeline_init, kbase_device_timeline_term, - "Timeline stream initialization failed" }, - { kbase_clk_rate_trace_manager_init, kbase_clk_rate_trace_manager_term, - "Clock rate trace manager initialization failed" }, - { kbase_instr_backend_init, kbase_instr_backend_term, - "Instrumentation backend initialization failed" }, - { kbase_device_hwcnt_watchdog_if_init, kbase_device_hwcnt_watchdog_if_term, - "GPU hwcnt backend watchdog interface creation failed" }, - { kbase_device_hwcnt_backend_jm_init, kbase_device_hwcnt_backend_jm_term, - "GPU hwcnt backend creation failed" }, - { kbase_device_hwcnt_backend_jm_watchdog_init, kbase_device_hwcnt_backend_jm_watchdog_term, - "GPU hwcnt watchdog backend creation failed" }, - { kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term, - "GPU hwcnt context initialization failed" }, - { kbase_device_hwcnt_virtualizer_init, kbase_device_hwcnt_virtualizer_term, - "GPU hwcnt virtualizer initialization failed" }, - { kbase_device_kinstr_prfcnt_init, kbase_device_kinstr_prfcnt_term, - "Performance counter instrumentation initialization failed" }, - { kbase_backend_late_init, kbase_backend_late_term, "Late backend initialization failed" }, - { kbase_debug_job_fault_dev_init, kbase_debug_job_fault_dev_term, - "Job fault debug initialization failed" }, - { kbase_device_debugfs_init, kbase_device_debugfs_term, "DebugFS initialization failed" }, - /* Sysfs init needs to happen before registering the device with - * misc_register(), otherwise it causes a race condition between - * registering the device and a uevent event being generated for - * userspace, causing udev rules to run which might expect certain - * sysfs attributes present. As a result of the race condition - * we avoid, some Mali sysfs entries may have appeared to udev - * to not exist. - * For more information, see - * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the - * paragraph that starts with "Word of warning", currently the - * second-last paragraph. - */ - { kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed" }, - { kbase_device_misc_register, kbase_device_misc_deregister, - "Misc device registration failed" }, - { kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer, - "GPU property population failed" }, - { NULL, kbase_dummy_job_wa_cleanup, NULL }, - { kbase_device_late_init, kbase_device_late_term, "Late device initialization failed" }, -}; - -static void kbase_device_term_partial(struct kbase_device *kbdev, unsigned int i) -{ - while (i-- > 0) { - if (dev_init[i].term) - dev_init[i].term(kbdev); - } -} - -void kbase_device_term(struct kbase_device *kbdev) -{ - kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init)); - kbasep_js_devdata_halt(kbdev); - kbase_mem_halt(kbdev); -} - -int kbase_device_init(struct kbase_device *kbdev) -{ - int err = 0; - unsigned int i = 0; - - dev_info(kbdev->dev, "Kernel DDK version %s", MALI_RELEASE_NAME); - - kbase_device_id_init(kbdev); - kbase_disjoint_init(kbdev); - - for (i = 0; i < ARRAY_SIZE(dev_init); i++) { - if (dev_init[i].init) { - err = dev_init[i].init(kbdev); - if (err) { - if (err != -EPROBE_DEFER) - dev_err(kbdev->dev, "%s error = %d\n", dev_init[i].err_mes, - err); - kbase_device_term_partial(kbdev, i); - break; - } - } - } - - return err; -} - -int kbase_device_firmware_init_once(struct kbase_device *kbdev) -{ - int ret = 0; - - mutex_lock(&kbdev->fw_load_lock); - - if (!kbdev->dummy_job_wa_loaded) { - ret = kbase_dummy_job_wa_load(kbdev); - if (!ret) - kbdev->dummy_job_wa_loaded = true; - } - - mutex_unlock(&kbdev->fw_load_lock); - - return ret; -} diff --git a/drivers/gpu/arm/valhall/device/mali_kbase_device.c b/drivers/gpu/arm/valhall/device/mali_kbase_device.c index 8de24066f2ee..78eab00ea46b 100644 --- a/drivers/gpu/arm/valhall/device/mali_kbase_device.c +++ b/drivers/gpu/arm/valhall/device/mali_kbase_device.c @@ -53,6 +53,12 @@ #include "arbiter/mali_kbase_arbiter_pm.h" #include +static uint64_t neural_allowed_mask = UINT64_MAX; +module_param(neural_allowed_mask, ullong, 0444); +MODULE_PARM_DESC( + neural_allowed_mask, + "Additional optional bitmask to restrict which neural engine cores any CSG can enable"); + #if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) /* Number of register accesses for the buffer that we allocate during @@ -123,8 +129,6 @@ static void kbase_device_all_as_term(struct kbase_device *kbdev) static int pcm_prioritized_process_cb(struct notifier_block *nb, unsigned long action, void *data) { -#if MALI_USE_CSF - struct kbase_device *const kbdev = container_of(nb, struct kbase_device, pcm_prioritized_process_nb); struct pcm_prioritized_process_notifier_data *const notifier_data = data; @@ -143,8 +147,6 @@ static int pcm_prioritized_process_cb(struct notifier_block *nb, unsigned long a return ret; -#endif /* MALI_USE_CSF */ - return 0; } @@ -351,12 +353,7 @@ int kbase_device_misc_init(struct kbase_device *const kbdev) kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; -#if MALI_USE_CSF kbdev->reset_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_GPU_RESET_TIMEOUT); -#else /* MALI_USE_CSF */ - kbdev->reset_timeout_ms = JM_DEFAULT_RESET_TIMEOUT_MS; -#endif /* !MALI_USE_CSF */ - kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); mutex_init(&kbdev->kctx_list_lock); @@ -372,15 +369,9 @@ int kbase_device_misc_init(struct kbase_device *const kbdev) kbdev->oom_notifier_block.notifier_call = NULL; } -#if !MALI_USE_CSF - spin_lock_init(&kbdev->quick_reset_lock); - kbdev->quick_reset_enabled = true; - kbdev->num_of_atoms_hw_completed = 0; -#endif - -#if MALI_USE_CSF atomic_set(&kbdev->fence_signal_timeout_enabled, 1); -#endif + + kbdev->csf.neural_allowed_mask = neural_allowed_mask; return 0; @@ -405,43 +396,12 @@ void kbase_device_misc_term(struct kbase_device *kbdev) if (kbdev->oom_notifier_block.notifier_call) unregister_oom_notifier(&kbdev->oom_notifier_block); -#if MALI_USE_CSF && IS_ENABLED(CONFIG_SYNC_FILE) +#if IS_ENABLED(CONFIG_SYNC_FILE) if (atomic_read(&kbdev->live_fence_metadata) > 0) dev_warn(kbdev->dev, "Terminating Kbase device with live fence metadata!"); #endif } -#if !MALI_USE_CSF -void kbase_enable_quick_reset(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->quick_reset_lock, flags); - - kbdev->quick_reset_enabled = true; - kbdev->num_of_atoms_hw_completed = 0; - - spin_unlock_irqrestore(&kbdev->quick_reset_lock, flags); -} - -void kbase_disable_quick_reset(struct kbase_device *kbdev) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->quick_reset_lock, flags); - - kbdev->quick_reset_enabled = false; - kbdev->num_of_atoms_hw_completed = 0; - - spin_unlock_irqrestore(&kbdev->quick_reset_lock, flags); -} - -bool kbase_is_quick_reset_enabled(struct kbase_device *kbdev) -{ - return kbdev->quick_reset_enabled; -} -#endif - void kbase_device_free(struct kbase_device *kbdev) { vfree(kbdev); diff --git a/drivers/gpu/arm/valhall/device/mali_kbase_device.h b/drivers/gpu/arm/valhall/device/mali_kbase_device.h index 32dff8c2ed1e..2c40fefffc04 100644 --- a/drivers/gpu/arm/valhall/device/mali_kbase_device.h +++ b/drivers/gpu/arm/valhall/device/mali_kbase_device.h @@ -102,10 +102,8 @@ void kbase_device_term(struct kbase_device *kbdev); * * Return: 0 if successful or a negative error code on failure. */ -#if MALI_USE_CSF int kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device *kbdev, phys_addr_t phys, size_t nr_bytes, u32 flush_op); -#endif /* MALI_USE_CSF */ /** * kbase_gpu_cache_flush_and_busy_wait - Start a cache flush and busy wait @@ -196,7 +194,6 @@ void kbase_clean_caches_done(struct kbase_device *kbdev); */ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); -#if MALI_USE_CSF /** * kbase_pwr_interrupt - GPU power interrupt handler * @kbdev: Kbase device pointer @@ -206,5 +203,4 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); * handled. */ void kbase_pwr_interrupt(struct kbase_device *kbdev, u32 val); -#endif /* MALI_USE_CSF */ #endif /* _MALI_KBASE_DEVICE_H_ */ diff --git a/drivers/gpu/arm/valhall/device/mali_kbase_device_hw.c b/drivers/gpu/arm/valhall/device/mali_kbase_device_hw.c index 3bfcc5e8f54e..5cfd10afa597 100644 --- a/drivers/gpu/arm/valhall/device/mali_kbase_device_hw.c +++ b/drivers/gpu/arm/valhall/device/mali_kbase_device_hw.c @@ -44,9 +44,7 @@ static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) bool completed = false; s64 diff; u32 irq_bits_to_check = irq_bit; -#if MALI_USE_CSF const bool has_host_pwr_iface = kbdev->pm.backend.has_host_pwr_iface; -#endif /* MALI_USE_CSF */ /* hwaccess_lock must be held to prevent concurrent threads from * cleaning the IRQ bits, otherwise it could be possible for this thread @@ -60,12 +58,8 @@ static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) * been reset which implies that any cache flush operation has been * completed, too. */ -#if MALI_USE_CSF if (!has_host_pwr_iface) -#endif /* MALI_USE_CSF */ - { irq_bits_to_check |= RESET_COMPLETED; - } do { unsigned int i; @@ -76,7 +70,6 @@ static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) completed = true; break; } -#if MALI_USE_CSF /* Check whether the GPU has been reset, which implies that any * cache flush operation has been completed. */ @@ -87,11 +80,8 @@ static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) break; } } -#endif /* MALI_USE_CSF */ } - if (kbase_io_is_aw_removed(kbdev)) - return -ENODEV; diff = ktime_to_ms(ktime_sub(ktime_get_raw(), wait_loop_start)); } while ((diff < wait_time_ms) && !completed); @@ -127,8 +117,6 @@ static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) return 0; } -#if MALI_USE_CSF - int kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device *kbdev, phys_addr_t phys, size_t nr_bytes, u32 flush_op) { @@ -137,6 +125,7 @@ int kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device *kbdev, phy lockdep_assert_held(&kbdev->hwaccess_lock); + /* 1. Clear the interrupt FLUSH_PA_RANGE_COMPLETED bit. */ kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_CLEAR), FLUSH_PA_RANGE_COMPLETED); @@ -153,7 +142,6 @@ int kbase_gpu_cache_flush_pa_range_and_busy_wait(struct kbase_device *kbdev, phy return ret; } -#endif /* MALI_USE_CSF */ int kbase_gpu_cache_flush_and_busy_wait(struct kbase_device *kbdev, u32 flush_op) { @@ -165,6 +153,7 @@ int kbase_gpu_cache_flush_and_busy_wait(struct kbase_device *kbdev, u32 flush_op */ lockdep_assert_held(&kbdev->hwaccess_lock); + /* 1. Check if kbdev->cache_clean_in_progress is set. * If it is set, it means there are threads waiting for * CLEAN_CACHES_COMPLETED irq to be raised and that the @@ -218,6 +207,7 @@ void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev, u32 flush_op lockdep_assert_held(&kbdev->hwaccess_lock); + if (kbdev->cache_clean_in_progress) { /* If this is called while another clean is in progress, we * can't rely on the current one to flush any new changes in diff --git a/drivers/gpu/arm/valhall/gpu/Kbuild b/drivers/gpu/arm/valhall/gpu/Kbuild index 772182586a24..cffa4edad1d8 100644 --- a/drivers/gpu/arm/valhall/gpu/Kbuild +++ b/drivers/gpu/arm/valhall/gpu/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012, 2020-2021, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -19,9 +19,4 @@ # valhall_kbase-y += gpu/mali_kbase_gpu.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += gpu/backend/mali_kbase_gpu_fault_csf.o -else - valhall_kbase-y += gpu/backend/mali_kbase_gpu_fault_jm.o -endif +valhall_kbase-y += gpu/backend/mali_kbase_gpu_fault_csf.o diff --git a/drivers/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_fault_jm.c b/drivers/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_fault_jm.c deleted file mode 100644 index 7f3743ca6432..000000000000 --- a/drivers/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_fault_jm.c +++ /dev/null @@ -1,176 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2022 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include - -#include - -const char *kbase_gpu_exception_name(u32 const exception_code) -{ - const char *e; - - switch (exception_code) { - /* Non-Fault Status code */ - case 0x00: - e = "NOT_STARTED/IDLE/OK"; - break; - case 0x01: - e = "DONE"; - break; - case 0x02: - e = "INTERRUPTED"; - break; - case 0x03: - e = "STOPPED"; - break; - case 0x04: - e = "TERMINATED"; - break; - case 0x08: - e = "ACTIVE"; - break; - /* Job exceptions */ - case 0x40: - e = "JOB_CONFIG_FAULT"; - break; - case 0x41: - e = "JOB_POWER_FAULT"; - break; - case 0x42: - e = "JOB_READ_FAULT"; - break; - case 0x43: - e = "JOB_WRITE_FAULT"; - break; - case 0x44: - e = "JOB_AFFINITY_FAULT"; - break; - case 0x48: - e = "JOB_BUS_FAULT"; - break; - case 0x50: - e = "INSTR_INVALID_PC"; - break; - case 0x51: - e = "INSTR_INVALID_ENC"; - break; - case 0x52: - e = "INSTR_TYPE_MISMATCH"; - break; - case 0x53: - e = "INSTR_OPERAND_FAULT"; - break; - case 0x54: - e = "INSTR_TLS_FAULT"; - break; - case 0x55: - e = "INSTR_BARRIER_FAULT"; - break; - case 0x56: - e = "INSTR_ALIGN_FAULT"; - break; - case 0x58: - e = "DATA_INVALID_FAULT"; - break; - case 0x59: - e = "TILE_RANGE_FAULT"; - break; - case 0x5A: - e = "ADDR_RANGE_FAULT"; - break; - case 0x60: - e = "OUT_OF_MEMORY"; - break; - /* GPU exceptions */ - case 0x80: - e = "DELAYED_BUS_FAULT"; - break; - case 0x88: - e = "SHAREABILITY_FAULT"; - break; - /* MMU exceptions */ - case 0xC0: - case 0xC1: - case 0xC2: - case 0xC3: - case 0xC4: - case 0xC5: - case 0xC6: - case 0xC7: - e = "TRANSLATION_FAULT"; - break; - case 0xC8: - case 0xC9: - case 0xCA: - case 0xCB: - case 0xCC: - case 0xCD: - case 0xCE: - case 0xCF: - e = "PERMISSION_FAULT"; - break; - case 0xD0: - case 0xD1: - case 0xD2: - case 0xD3: - case 0xD4: - case 0xD5: - case 0xD6: - case 0xD7: - e = "TRANSTAB_BUS_FAULT"; - break; - case 0xD8: - case 0xD9: - case 0xDA: - case 0xDB: - case 0xDC: - case 0xDD: - case 0xDE: - case 0xDF: - e = "ACCESS_FLAG"; - break; - case 0xE0: - case 0xE1: - case 0xE2: - case 0xE3: - case 0xE4: - case 0xE5: - case 0xE6: - case 0xE7: - e = "ADDRESS_SIZE_FAULT"; - break; - case 0xE8: - case 0xE9: - case 0xEA: - case 0xEB: - case 0xEC: - case 0xED: - case 0xEE: - case 0xEF: - e = "MEMORY_ATTRIBUTES_FAULT"; - break; - default: - e = "UNKNOWN"; - break; - } - - return e; -} diff --git a/drivers/gpu/arm/valhall/hw_access/Kbuild b/drivers/gpu/arm/valhall/hw_access/Kbuild index f3bbfc8bafe2..0d3d45fb0900 100644 --- a/drivers/gpu/arm/valhall/hw_access/Kbuild +++ b/drivers/gpu/arm/valhall/hw_access/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -19,12 +19,7 @@ # valhall_kbase-y += hw_access/mali_kbase_hw_access.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += hw_access/regmap/mali_kbase_regmap_csf.o -else - valhall_kbase-y += hw_access/regmap/mali_kbase_regmap_jm.o -endif +valhall_kbase-y += hw_access/regmap/mali_kbase_regmap_csf.o ifeq ($(CONFIG_MALI_VALHALL_REAL_HW), y) valhall_kbase-y += hw_access/backend/mali_kbase_hw_access_real_hw.o diff --git a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.c b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.c index f06a62b607c6..45fb2e65640d 100644 --- a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.c +++ b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.c @@ -33,19 +33,6 @@ static u32 always_powered_regs[] = { -#if !MALI_USE_CSF - PTM_AW_IRQ_CLEAR, - PTM_AW_IRQ_INJECTION, - PTM_AW_IRQ_MASK, - PTM_AW_IRQ_RAWSTAT, - PTM_AW_IRQ_STATUS, - PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0, - PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1, - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0, - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1, - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS, - PTM_ID, -#endif /* !MALI_USE_CSF */ }; static void kbasep_reg_setup_always_powered_registers(struct kbase_device *kbdev) @@ -53,11 +40,6 @@ static void kbasep_reg_setup_always_powered_registers(struct kbase_device *kbdev u32 i; -#if !MALI_USE_CSF - if (kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(9, 14, 0)) - return; -#endif /* MALI_USE_CSF */ - for (i = 0; i < ARRAY_SIZE(always_powered_regs); i++) { u32 reg_enum = always_powered_regs[i]; diff --git a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.h b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.h index 1e9f016e2969..3aa50b0770a3 100644 --- a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.h +++ b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access.h @@ -270,9 +270,7 @@ static inline u32 kbase_reg_gpu_irq_all(bool is_legacy) u32 mask = GPU_IRQ_REG_COMMON; if (is_legacy) { -#if MALI_USE_CSF mask |= (RESET_COMPLETED | POWER_CHANGED_ALL); -#endif /* MALI_USE_CSF */ /* Include POWER_CHANGED_SINGLE in debug builds for use in irq latency test. */ if (IS_ENABLED(CONFIG_MALI_VALHALL_DEBUG)) mask |= POWER_CHANGED_SINGLE; diff --git a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap.h b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap.h index c10a32de1e91..bf00919e720a 100644 --- a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap.h +++ b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap.h @@ -22,13 +22,8 @@ #ifndef _MALI_KBASE_HW_ACCESS_REGMAP_H_ #define _MALI_KBASE_HW_ACCESS_REGMAP_H_ -#if MALI_USE_CSF #include "regmap/mali_kbase_regmap_csf_enums.h" #include "regmap/mali_kbase_regmap_csf_macros.h" -#else -#include "regmap/mali_kbase_regmap_jm_enums.h" -#include "regmap/mali_kbase_regmap_jm_macros.h" -#endif /* GPU_U definition */ #ifdef __ASSEMBLER__ diff --git a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap_legacy.h b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap_legacy.h index 1722428a7904..d4b0b6ee86d3 100644 --- a/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap_legacy.h +++ b/drivers/gpu/arm/valhall/hw_access/mali_kbase_hw_access_regmap_legacy.h @@ -22,11 +22,7 @@ #ifndef _MALI_KBASE_REGMAP_LEGACY_H_ #define _MALI_KBASE_REGMAP_LEGACY_H_ -#if MALI_USE_CSF #include "regmap/mali_kbase_regmap_legacy_csf.h" -#else -#include "regmap/mali_kbase_regmap_legacy_jm.h" -#endif /* Begin Register Offsets */ diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_enums.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_enums.h index a912c1fd095a..300781384fe9 100644 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_enums.h +++ b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_enums.h @@ -27,10 +27,6 @@ #ifndef _MALI_KBASE_REGMAP_CSF_ENUMS_H_ #define _MALI_KBASE_REGMAP_CSF_ENUMS_H_ -#if !MALI_USE_CSF -#error "Cannot be compiled with JM" -#endif - enum kbase_regmap_enum_v10_8 { GPU_CONTROL__GPU_ID = 0, /* (RO) 32-bit 0x0 */ GPU_CONTROL__L2_FEATURES, /* (RO) 32-bit 0x4 */ diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_macros.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_macros.h index afdd0dfd44c2..121312acef24 100644 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_macros.h +++ b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_csf_macros.h @@ -22,10 +22,6 @@ #ifndef _MALI_KBASE_REGMAP_CSF_MACROS_H_ #define _MALI_KBASE_REGMAP_CSF_MACROS_H_ -#if !MALI_USE_CSF -#error "Cannot be compiled with JM" -#endif - #define ENUM_OFFSET(_index, _base, _next) (_base + _index * (_next - _base)) diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm.c b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm.c deleted file mode 100644 index 819fae7ed398..000000000000 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm.c +++ /dev/null @@ -1,3041 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * This file is autogenerated. Avoid modifying this file unless absolutely - * necessary. - */ - -#include -#include - -#include -#include "mali_kbase_regmap_jm_enums.h" -#include - -static void kbase_regmap_v6_0_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V6_0_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbdev->regmap.flags[GPU_CONTROL__GPU_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__L2_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TILER_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__MEM_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__MMU_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__AS_PRESENT] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS_PRESENT] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__GPU_IRQ_RAWSTAT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__GPU_IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__GPU_IRQ_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__GPU_IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__GPU_COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__GPU_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__LATEST_FLUSH] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__GPU_FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__GPU_FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__AFBC_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__PWR_KEY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PWR_OVERRIDE0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PWR_OVERRIDE1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_BASE] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_JM_EN] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_SHADER_EN] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_TILER_EN] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__PRFCNT_MMU_L2_EN] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__CYCLE_COUNT] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TIMESTAMP] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__THREAD_MAX_THREADS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__THREAD_MAX_WORKGROUP_SIZE] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__THREAD_MAX_BARRIER_SIZE] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__THREAD_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TEXTURE_FEATURES_0] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TEXTURE_FEATURES_1] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TEXTURE_FEATURES_2] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS0_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS1_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS2_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS3_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS4_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS5_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS6_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS7_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS8_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS9_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS10_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS11_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS12_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS13_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS14_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__JS15_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__SHADER_PRESENT] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TILER_PRESENT] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__L2_PRESENT] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__SHADER_READY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TILER_READY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__L2_READY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__SHADER_PWRON] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__TILER_PWRON] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__L2_PWRON] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__SHADER_PWROFF] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__TILER_PWROFF] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__L2_PWROFF] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__SHADER_PWRTRANS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TILER_PWRTRANS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__L2_PWRTRANS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__SHADER_PWRACTIVE] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__TILER_PWRACTIVE] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__L2_PWRACTIVE] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__COHERENCY_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__COHERENCY_ENABLE] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__GPU_USER_INn] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__GPU_USER_OUTn] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__JM_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__SHADER_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__TILER_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__L2_MMU_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_RAWSTAT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_JS_STATE] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_THROTTLE] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__EVENT_IRQ_RAWSTAT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__EVENT_IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__EVENT_IRQ_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__EVENT_IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS0__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS0__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS1__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS1__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS2__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS2__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS3__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS3__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS4__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS4__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS5__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS5__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS6__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS6__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS7__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS7__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS8__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS8__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS9__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS9__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS10__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS10__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS11__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS11__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS12__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS12__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS13__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS13__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS14__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS14__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__HEAD] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__TAIL] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__AFFINITY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__CONFIG] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__XAFFINITY] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__COMMAND] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__FLUSH_ID] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[JOB_CONTROL__JS15__EVENT_0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__EVENT_1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__EVENT_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__HEAD_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__TAIL_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__AFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__CONFIG_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__XAFFINITY_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__COMMAND_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__FLUSH_ID_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[JOB_CONTROL__JS15__EVENT_MASK_NEXT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__IRQ_RAWSTAT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__IRQ_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS0__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS1__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS2__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS3__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS4__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS5__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS6__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS7__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS8__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS9__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS10__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS11__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS12__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS13__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS14__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE1__ST1MMU__AS15__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__IRQ_RAWSTAT] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__IRQ_MASK] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS0__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS1__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS2__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS3__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS4__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS5__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS6__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS7__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS8__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS9__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS10__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS11__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS12__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS13__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS14__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__TRANSTAB] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__MEMATTR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__LOCKADDR] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__COMMAND] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__FAULTSTATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__FAULTADDRESS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__STATUS] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__TRANSCFG] = - KBASE_REGMAP_WIDTH_64_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[MMU_STAGE2__ST2MMU__AS15__FAULTEXTRA] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - - kbdev->regmap.regs[GPU_CONTROL__GPU_ID] = kbdev->reg + 0x0; - kbdev->regmap.regs[GPU_CONTROL__L2_FEATURES] = kbdev->reg + 0x4; - kbdev->regmap.regs[GPU_CONTROL__TILER_FEATURES] = kbdev->reg + 0xc; - kbdev->regmap.regs[GPU_CONTROL__MEM_FEATURES] = kbdev->reg + 0x10; - kbdev->regmap.regs[GPU_CONTROL__MMU_FEATURES] = kbdev->reg + 0x14; - kbdev->regmap.regs[GPU_CONTROL__AS_PRESENT] = kbdev->reg + 0x18; - kbdev->regmap.regs[GPU_CONTROL__JS_PRESENT] = kbdev->reg + 0x1c; - kbdev->regmap.regs[GPU_CONTROL__GPU_IRQ_RAWSTAT] = kbdev->reg + 0x20; - kbdev->regmap.regs[GPU_CONTROL__GPU_IRQ_CLEAR] = kbdev->reg + 0x24; - kbdev->regmap.regs[GPU_CONTROL__GPU_IRQ_MASK] = kbdev->reg + 0x28; - kbdev->regmap.regs[GPU_CONTROL__GPU_IRQ_STATUS] = kbdev->reg + 0x2c; - kbdev->regmap.regs[GPU_CONTROL__GPU_COMMAND] = kbdev->reg + 0x30; - kbdev->regmap.regs[GPU_CONTROL__GPU_STATUS] = kbdev->reg + 0x34; - kbdev->regmap.regs[GPU_CONTROL__LATEST_FLUSH] = kbdev->reg + 0x38; - kbdev->regmap.regs[GPU_CONTROL__GPU_FAULTSTATUS] = kbdev->reg + 0x3c; - kbdev->regmap.regs[GPU_CONTROL__GPU_FAULTADDRESS] = kbdev->reg + 0x40; - kbdev->regmap.regs[GPU_CONTROL__AFBC_FEATURES] = kbdev->reg + 0x4c; - kbdev->regmap.regs[GPU_CONTROL__PWR_KEY] = kbdev->reg + 0x50; - kbdev->regmap.regs[GPU_CONTROL__PWR_OVERRIDE0] = kbdev->reg + 0x54; - kbdev->regmap.regs[GPU_CONTROL__PWR_OVERRIDE1] = kbdev->reg + 0x58; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_BASE] = kbdev->reg + 0x60; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_CONFIG] = kbdev->reg + 0x68; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_JM_EN] = kbdev->reg + 0x6c; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_SHADER_EN] = kbdev->reg + 0x70; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_TILER_EN] = kbdev->reg + 0x74; - kbdev->regmap.regs[GPU_CONTROL__PRFCNT_MMU_L2_EN] = kbdev->reg + 0x7c; - kbdev->regmap.regs[GPU_CONTROL__CYCLE_COUNT] = kbdev->reg + 0x90; - kbdev->regmap.regs[GPU_CONTROL__TIMESTAMP] = kbdev->reg + 0x98; - kbdev->regmap.regs[GPU_CONTROL__THREAD_MAX_THREADS] = kbdev->reg + 0xa0; - kbdev->regmap.regs[GPU_CONTROL__THREAD_MAX_WORKGROUP_SIZE] = kbdev->reg + 0xa4; - kbdev->regmap.regs[GPU_CONTROL__THREAD_MAX_BARRIER_SIZE] = kbdev->reg + 0xa8; - kbdev->regmap.regs[GPU_CONTROL__THREAD_FEATURES] = kbdev->reg + 0xac; - kbdev->regmap.regs[GPU_CONTROL__TEXTURE_FEATURES_0] = kbdev->reg + 0xb0; - kbdev->regmap.regs[GPU_CONTROL__TEXTURE_FEATURES_1] = kbdev->reg + 0xb4; - kbdev->regmap.regs[GPU_CONTROL__TEXTURE_FEATURES_2] = kbdev->reg + 0xb8; - kbdev->regmap.regs[GPU_CONTROL__JS0_FEATURES] = kbdev->reg + 0xc0; - kbdev->regmap.regs[GPU_CONTROL__JS1_FEATURES] = kbdev->reg + 0xc4; - kbdev->regmap.regs[GPU_CONTROL__JS2_FEATURES] = kbdev->reg + 0xc8; - kbdev->regmap.regs[GPU_CONTROL__JS3_FEATURES] = kbdev->reg + 0xcc; - kbdev->regmap.regs[GPU_CONTROL__JS4_FEATURES] = kbdev->reg + 0xd0; - kbdev->regmap.regs[GPU_CONTROL__JS5_FEATURES] = kbdev->reg + 0xd4; - kbdev->regmap.regs[GPU_CONTROL__JS6_FEATURES] = kbdev->reg + 0xd8; - kbdev->regmap.regs[GPU_CONTROL__JS7_FEATURES] = kbdev->reg + 0xdc; - kbdev->regmap.regs[GPU_CONTROL__JS8_FEATURES] = kbdev->reg + 0xe0; - kbdev->regmap.regs[GPU_CONTROL__JS9_FEATURES] = kbdev->reg + 0xe4; - kbdev->regmap.regs[GPU_CONTROL__JS10_FEATURES] = kbdev->reg + 0xe8; - kbdev->regmap.regs[GPU_CONTROL__JS11_FEATURES] = kbdev->reg + 0xec; - kbdev->regmap.regs[GPU_CONTROL__JS12_FEATURES] = kbdev->reg + 0xf0; - kbdev->regmap.regs[GPU_CONTROL__JS13_FEATURES] = kbdev->reg + 0xf4; - kbdev->regmap.regs[GPU_CONTROL__JS14_FEATURES] = kbdev->reg + 0xf8; - kbdev->regmap.regs[GPU_CONTROL__JS15_FEATURES] = kbdev->reg + 0xfc; - kbdev->regmap.regs[GPU_CONTROL__SHADER_PRESENT] = kbdev->reg + 0x100; - kbdev->regmap.regs[GPU_CONTROL__TILER_PRESENT] = kbdev->reg + 0x110; - kbdev->regmap.regs[GPU_CONTROL__L2_PRESENT] = kbdev->reg + 0x120; - kbdev->regmap.regs[GPU_CONTROL__SHADER_READY] = kbdev->reg + 0x140; - kbdev->regmap.regs[GPU_CONTROL__TILER_READY] = kbdev->reg + 0x150; - kbdev->regmap.regs[GPU_CONTROL__L2_READY] = kbdev->reg + 0x160; - kbdev->regmap.regs[GPU_CONTROL__SHADER_PWRON] = kbdev->reg + 0x180; - kbdev->regmap.regs[GPU_CONTROL__TILER_PWRON] = kbdev->reg + 0x190; - kbdev->regmap.regs[GPU_CONTROL__L2_PWRON] = kbdev->reg + 0x1a0; - kbdev->regmap.regs[GPU_CONTROL__SHADER_PWROFF] = kbdev->reg + 0x1c0; - kbdev->regmap.regs[GPU_CONTROL__TILER_PWROFF] = kbdev->reg + 0x1d0; - kbdev->regmap.regs[GPU_CONTROL__L2_PWROFF] = kbdev->reg + 0x1e0; - kbdev->regmap.regs[GPU_CONTROL__SHADER_PWRTRANS] = kbdev->reg + 0x200; - kbdev->regmap.regs[GPU_CONTROL__TILER_PWRTRANS] = kbdev->reg + 0x210; - kbdev->regmap.regs[GPU_CONTROL__L2_PWRTRANS] = kbdev->reg + 0x220; - kbdev->regmap.regs[GPU_CONTROL__SHADER_PWRACTIVE] = kbdev->reg + 0x240; - kbdev->regmap.regs[GPU_CONTROL__TILER_PWRACTIVE] = kbdev->reg + 0x250; - kbdev->regmap.regs[GPU_CONTROL__L2_PWRACTIVE] = kbdev->reg + 0x260; - kbdev->regmap.regs[GPU_CONTROL__COHERENCY_FEATURES] = kbdev->reg + 0x300; - kbdev->regmap.regs[GPU_CONTROL__COHERENCY_ENABLE] = kbdev->reg + 0x304; - kbdev->regmap.regs[GPU_CONTROL__GPU_USER_INn] = kbdev->reg + 0x400; - kbdev->regmap.regs[GPU_CONTROL__GPU_USER_OUTn] = kbdev->reg + 0x500; - kbdev->regmap.regs[GPU_CONTROL__JM_CONFIG] = kbdev->reg + 0xf00; - kbdev->regmap.regs[GPU_CONTROL__SHADER_CONFIG] = kbdev->reg + 0xf04; - kbdev->regmap.regs[GPU_CONTROL__TILER_CONFIG] = kbdev->reg + 0xf08; - kbdev->regmap.regs[GPU_CONTROL__L2_MMU_CONFIG] = kbdev->reg + 0xf0c; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_RAWSTAT] = kbdev->reg + 0x1000; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_CLEAR] = kbdev->reg + 0x1004; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_MASK] = kbdev->reg + 0x1008; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_STATUS] = kbdev->reg + 0x100c; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_JS_STATE] = kbdev->reg + 0x1010; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_THROTTLE] = kbdev->reg + 0x1014; - kbdev->regmap.regs[JOB_CONTROL__EVENT_IRQ_RAWSTAT] = kbdev->reg + 0x1020; - kbdev->regmap.regs[JOB_CONTROL__EVENT_IRQ_CLEAR] = kbdev->reg + 0x1024; - kbdev->regmap.regs[JOB_CONTROL__EVENT_IRQ_MASK] = kbdev->reg + 0x1028; - kbdev->regmap.regs[JOB_CONTROL__EVENT_IRQ_STATUS] = kbdev->reg + 0x102c; - kbdev->regmap.regs[JOB_CONTROL__JS0__HEAD] = kbdev->reg + 0x1800; - kbdev->regmap.regs[JOB_CONTROL__JS0__TAIL] = kbdev->reg + 0x1808; - kbdev->regmap.regs[JOB_CONTROL__JS0__AFFINITY] = kbdev->reg + 0x1810; - kbdev->regmap.regs[JOB_CONTROL__JS0__CONFIG] = kbdev->reg + 0x1818; - kbdev->regmap.regs[JOB_CONTROL__JS0__XAFFINITY] = kbdev->reg + 0x181c; - kbdev->regmap.regs[JOB_CONTROL__JS0__COMMAND] = kbdev->reg + 0x1820; - kbdev->regmap.regs[JOB_CONTROL__JS0__STATUS] = kbdev->reg + 0x1824; - kbdev->regmap.regs[JOB_CONTROL__JS0__FLUSH_ID] = kbdev->reg + 0x1830; - kbdev->regmap.regs[JOB_CONTROL__JS0__EVENT_0] = kbdev->reg + 0x1834; - kbdev->regmap.regs[JOB_CONTROL__JS0__EVENT_1] = kbdev->reg + 0x1838; - kbdev->regmap.regs[JOB_CONTROL__JS0__EVENT_MASK] = kbdev->reg + 0x183c; - kbdev->regmap.regs[JOB_CONTROL__JS0__HEAD_NEXT] = kbdev->reg + 0x1840; - kbdev->regmap.regs[JOB_CONTROL__JS0__TAIL_NEXT] = kbdev->reg + 0x1848; - kbdev->regmap.regs[JOB_CONTROL__JS0__AFFINITY_NEXT] = kbdev->reg + 0x1850; - kbdev->regmap.regs[JOB_CONTROL__JS0__CONFIG_NEXT] = kbdev->reg + 0x1858; - kbdev->regmap.regs[JOB_CONTROL__JS0__XAFFINITY_NEXT] = kbdev->reg + 0x185c; - kbdev->regmap.regs[JOB_CONTROL__JS0__COMMAND_NEXT] = kbdev->reg + 0x1860; - kbdev->regmap.regs[JOB_CONTROL__JS0__FLUSH_ID_NEXT] = kbdev->reg + 0x1870; - kbdev->regmap.regs[JOB_CONTROL__JS0__EVENT_MASK_NEXT] = kbdev->reg + 0x187c; - kbdev->regmap.regs[JOB_CONTROL__JS1__HEAD] = kbdev->reg + 0x1880; - kbdev->regmap.regs[JOB_CONTROL__JS1__TAIL] = kbdev->reg + 0x1888; - kbdev->regmap.regs[JOB_CONTROL__JS1__AFFINITY] = kbdev->reg + 0x1890; - kbdev->regmap.regs[JOB_CONTROL__JS1__CONFIG] = kbdev->reg + 0x1898; - kbdev->regmap.regs[JOB_CONTROL__JS1__XAFFINITY] = kbdev->reg + 0x189c; - kbdev->regmap.regs[JOB_CONTROL__JS1__COMMAND] = kbdev->reg + 0x18a0; - kbdev->regmap.regs[JOB_CONTROL__JS1__STATUS] = kbdev->reg + 0x18a4; - kbdev->regmap.regs[JOB_CONTROL__JS1__FLUSH_ID] = kbdev->reg + 0x18b0; - kbdev->regmap.regs[JOB_CONTROL__JS1__EVENT_0] = kbdev->reg + 0x18b4; - kbdev->regmap.regs[JOB_CONTROL__JS1__EVENT_1] = kbdev->reg + 0x18b8; - kbdev->regmap.regs[JOB_CONTROL__JS1__EVENT_MASK] = kbdev->reg + 0x18bc; - kbdev->regmap.regs[JOB_CONTROL__JS1__HEAD_NEXT] = kbdev->reg + 0x18c0; - kbdev->regmap.regs[JOB_CONTROL__JS1__TAIL_NEXT] = kbdev->reg + 0x18c8; - kbdev->regmap.regs[JOB_CONTROL__JS1__AFFINITY_NEXT] = kbdev->reg + 0x18d0; - kbdev->regmap.regs[JOB_CONTROL__JS1__CONFIG_NEXT] = kbdev->reg + 0x18d8; - kbdev->regmap.regs[JOB_CONTROL__JS1__XAFFINITY_NEXT] = kbdev->reg + 0x18dc; - kbdev->regmap.regs[JOB_CONTROL__JS1__COMMAND_NEXT] = kbdev->reg + 0x18e0; - kbdev->regmap.regs[JOB_CONTROL__JS1__FLUSH_ID_NEXT] = kbdev->reg + 0x18f0; - kbdev->regmap.regs[JOB_CONTROL__JS1__EVENT_MASK_NEXT] = kbdev->reg + 0x18fc; - kbdev->regmap.regs[JOB_CONTROL__JS2__HEAD] = kbdev->reg + 0x1900; - kbdev->regmap.regs[JOB_CONTROL__JS2__TAIL] = kbdev->reg + 0x1908; - kbdev->regmap.regs[JOB_CONTROL__JS2__AFFINITY] = kbdev->reg + 0x1910; - kbdev->regmap.regs[JOB_CONTROL__JS2__CONFIG] = kbdev->reg + 0x1918; - kbdev->regmap.regs[JOB_CONTROL__JS2__XAFFINITY] = kbdev->reg + 0x191c; - kbdev->regmap.regs[JOB_CONTROL__JS2__COMMAND] = kbdev->reg + 0x1920; - kbdev->regmap.regs[JOB_CONTROL__JS2__STATUS] = kbdev->reg + 0x1924; - kbdev->regmap.regs[JOB_CONTROL__JS2__FLUSH_ID] = kbdev->reg + 0x1930; - kbdev->regmap.regs[JOB_CONTROL__JS2__EVENT_0] = kbdev->reg + 0x1934; - kbdev->regmap.regs[JOB_CONTROL__JS2__EVENT_1] = kbdev->reg + 0x1938; - kbdev->regmap.regs[JOB_CONTROL__JS2__EVENT_MASK] = kbdev->reg + 0x193c; - kbdev->regmap.regs[JOB_CONTROL__JS2__HEAD_NEXT] = kbdev->reg + 0x1940; - kbdev->regmap.regs[JOB_CONTROL__JS2__TAIL_NEXT] = kbdev->reg + 0x1948; - kbdev->regmap.regs[JOB_CONTROL__JS2__AFFINITY_NEXT] = kbdev->reg + 0x1950; - kbdev->regmap.regs[JOB_CONTROL__JS2__CONFIG_NEXT] = kbdev->reg + 0x1958; - kbdev->regmap.regs[JOB_CONTROL__JS2__XAFFINITY_NEXT] = kbdev->reg + 0x195c; - kbdev->regmap.regs[JOB_CONTROL__JS2__COMMAND_NEXT] = kbdev->reg + 0x1960; - kbdev->regmap.regs[JOB_CONTROL__JS2__FLUSH_ID_NEXT] = kbdev->reg + 0x1970; - kbdev->regmap.regs[JOB_CONTROL__JS2__EVENT_MASK_NEXT] = kbdev->reg + 0x197c; - kbdev->regmap.regs[JOB_CONTROL__JS3__HEAD] = kbdev->reg + 0x1980; - kbdev->regmap.regs[JOB_CONTROL__JS3__TAIL] = kbdev->reg + 0x1988; - kbdev->regmap.regs[JOB_CONTROL__JS3__AFFINITY] = kbdev->reg + 0x1990; - kbdev->regmap.regs[JOB_CONTROL__JS3__CONFIG] = kbdev->reg + 0x1998; - kbdev->regmap.regs[JOB_CONTROL__JS3__XAFFINITY] = kbdev->reg + 0x199c; - kbdev->regmap.regs[JOB_CONTROL__JS3__COMMAND] = kbdev->reg + 0x19a0; - kbdev->regmap.regs[JOB_CONTROL__JS3__STATUS] = kbdev->reg + 0x19a4; - kbdev->regmap.regs[JOB_CONTROL__JS3__FLUSH_ID] = kbdev->reg + 0x19b0; - kbdev->regmap.regs[JOB_CONTROL__JS3__EVENT_0] = kbdev->reg + 0x19b4; - kbdev->regmap.regs[JOB_CONTROL__JS3__EVENT_1] = kbdev->reg + 0x19b8; - kbdev->regmap.regs[JOB_CONTROL__JS3__EVENT_MASK] = kbdev->reg + 0x19bc; - kbdev->regmap.regs[JOB_CONTROL__JS3__HEAD_NEXT] = kbdev->reg + 0x19c0; - kbdev->regmap.regs[JOB_CONTROL__JS3__TAIL_NEXT] = kbdev->reg + 0x19c8; - kbdev->regmap.regs[JOB_CONTROL__JS3__AFFINITY_NEXT] = kbdev->reg + 0x19d0; - kbdev->regmap.regs[JOB_CONTROL__JS3__CONFIG_NEXT] = kbdev->reg + 0x19d8; - kbdev->regmap.regs[JOB_CONTROL__JS3__XAFFINITY_NEXT] = kbdev->reg + 0x19dc; - kbdev->regmap.regs[JOB_CONTROL__JS3__COMMAND_NEXT] = kbdev->reg + 0x19e0; - kbdev->regmap.regs[JOB_CONTROL__JS3__FLUSH_ID_NEXT] = kbdev->reg + 0x19f0; - kbdev->regmap.regs[JOB_CONTROL__JS3__EVENT_MASK_NEXT] = kbdev->reg + 0x19fc; - kbdev->regmap.regs[JOB_CONTROL__JS4__HEAD] = kbdev->reg + 0x1a00; - kbdev->regmap.regs[JOB_CONTROL__JS4__TAIL] = kbdev->reg + 0x1a08; - kbdev->regmap.regs[JOB_CONTROL__JS4__AFFINITY] = kbdev->reg + 0x1a10; - kbdev->regmap.regs[JOB_CONTROL__JS4__CONFIG] = kbdev->reg + 0x1a18; - kbdev->regmap.regs[JOB_CONTROL__JS4__XAFFINITY] = kbdev->reg + 0x1a1c; - kbdev->regmap.regs[JOB_CONTROL__JS4__COMMAND] = kbdev->reg + 0x1a20; - kbdev->regmap.regs[JOB_CONTROL__JS4__STATUS] = kbdev->reg + 0x1a24; - kbdev->regmap.regs[JOB_CONTROL__JS4__FLUSH_ID] = kbdev->reg + 0x1a30; - kbdev->regmap.regs[JOB_CONTROL__JS4__EVENT_0] = kbdev->reg + 0x1a34; - kbdev->regmap.regs[JOB_CONTROL__JS4__EVENT_1] = kbdev->reg + 0x1a38; - kbdev->regmap.regs[JOB_CONTROL__JS4__EVENT_MASK] = kbdev->reg + 0x1a3c; - kbdev->regmap.regs[JOB_CONTROL__JS4__HEAD_NEXT] = kbdev->reg + 0x1a40; - kbdev->regmap.regs[JOB_CONTROL__JS4__TAIL_NEXT] = kbdev->reg + 0x1a48; - kbdev->regmap.regs[JOB_CONTROL__JS4__AFFINITY_NEXT] = kbdev->reg + 0x1a50; - kbdev->regmap.regs[JOB_CONTROL__JS4__CONFIG_NEXT] = kbdev->reg + 0x1a58; - kbdev->regmap.regs[JOB_CONTROL__JS4__XAFFINITY_NEXT] = kbdev->reg + 0x1a5c; - kbdev->regmap.regs[JOB_CONTROL__JS4__COMMAND_NEXT] = kbdev->reg + 0x1a60; - kbdev->regmap.regs[JOB_CONTROL__JS4__FLUSH_ID_NEXT] = kbdev->reg + 0x1a70; - kbdev->regmap.regs[JOB_CONTROL__JS4__EVENT_MASK_NEXT] = kbdev->reg + 0x1a7c; - kbdev->regmap.regs[JOB_CONTROL__JS5__HEAD] = kbdev->reg + 0x1a80; - kbdev->regmap.regs[JOB_CONTROL__JS5__TAIL] = kbdev->reg + 0x1a88; - kbdev->regmap.regs[JOB_CONTROL__JS5__AFFINITY] = kbdev->reg + 0x1a90; - kbdev->regmap.regs[JOB_CONTROL__JS5__CONFIG] = kbdev->reg + 0x1a98; - kbdev->regmap.regs[JOB_CONTROL__JS5__XAFFINITY] = kbdev->reg + 0x1a9c; - kbdev->regmap.regs[JOB_CONTROL__JS5__COMMAND] = kbdev->reg + 0x1aa0; - kbdev->regmap.regs[JOB_CONTROL__JS5__STATUS] = kbdev->reg + 0x1aa4; - kbdev->regmap.regs[JOB_CONTROL__JS5__FLUSH_ID] = kbdev->reg + 0x1ab0; - kbdev->regmap.regs[JOB_CONTROL__JS5__EVENT_0] = kbdev->reg + 0x1ab4; - kbdev->regmap.regs[JOB_CONTROL__JS5__EVENT_1] = kbdev->reg + 0x1ab8; - kbdev->regmap.regs[JOB_CONTROL__JS5__EVENT_MASK] = kbdev->reg + 0x1abc; - kbdev->regmap.regs[JOB_CONTROL__JS5__HEAD_NEXT] = kbdev->reg + 0x1ac0; - kbdev->regmap.regs[JOB_CONTROL__JS5__TAIL_NEXT] = kbdev->reg + 0x1ac8; - kbdev->regmap.regs[JOB_CONTROL__JS5__AFFINITY_NEXT] = kbdev->reg + 0x1ad0; - kbdev->regmap.regs[JOB_CONTROL__JS5__CONFIG_NEXT] = kbdev->reg + 0x1ad8; - kbdev->regmap.regs[JOB_CONTROL__JS5__XAFFINITY_NEXT] = kbdev->reg + 0x1adc; - kbdev->regmap.regs[JOB_CONTROL__JS5__COMMAND_NEXT] = kbdev->reg + 0x1ae0; - kbdev->regmap.regs[JOB_CONTROL__JS5__FLUSH_ID_NEXT] = kbdev->reg + 0x1af0; - kbdev->regmap.regs[JOB_CONTROL__JS5__EVENT_MASK_NEXT] = kbdev->reg + 0x1afc; - kbdev->regmap.regs[JOB_CONTROL__JS6__HEAD] = kbdev->reg + 0x1b00; - kbdev->regmap.regs[JOB_CONTROL__JS6__TAIL] = kbdev->reg + 0x1b08; - kbdev->regmap.regs[JOB_CONTROL__JS6__AFFINITY] = kbdev->reg + 0x1b10; - kbdev->regmap.regs[JOB_CONTROL__JS6__CONFIG] = kbdev->reg + 0x1b18; - kbdev->regmap.regs[JOB_CONTROL__JS6__XAFFINITY] = kbdev->reg + 0x1b1c; - kbdev->regmap.regs[JOB_CONTROL__JS6__COMMAND] = kbdev->reg + 0x1b20; - kbdev->regmap.regs[JOB_CONTROL__JS6__STATUS] = kbdev->reg + 0x1b24; - kbdev->regmap.regs[JOB_CONTROL__JS6__FLUSH_ID] = kbdev->reg + 0x1b30; - kbdev->regmap.regs[JOB_CONTROL__JS6__EVENT_0] = kbdev->reg + 0x1b34; - kbdev->regmap.regs[JOB_CONTROL__JS6__EVENT_1] = kbdev->reg + 0x1b38; - kbdev->regmap.regs[JOB_CONTROL__JS6__EVENT_MASK] = kbdev->reg + 0x1b3c; - kbdev->regmap.regs[JOB_CONTROL__JS6__HEAD_NEXT] = kbdev->reg + 0x1b40; - kbdev->regmap.regs[JOB_CONTROL__JS6__TAIL_NEXT] = kbdev->reg + 0x1b48; - kbdev->regmap.regs[JOB_CONTROL__JS6__AFFINITY_NEXT] = kbdev->reg + 0x1b50; - kbdev->regmap.regs[JOB_CONTROL__JS6__CONFIG_NEXT] = kbdev->reg + 0x1b58; - kbdev->regmap.regs[JOB_CONTROL__JS6__XAFFINITY_NEXT] = kbdev->reg + 0x1b5c; - kbdev->regmap.regs[JOB_CONTROL__JS6__COMMAND_NEXT] = kbdev->reg + 0x1b60; - kbdev->regmap.regs[JOB_CONTROL__JS6__FLUSH_ID_NEXT] = kbdev->reg + 0x1b70; - kbdev->regmap.regs[JOB_CONTROL__JS6__EVENT_MASK_NEXT] = kbdev->reg + 0x1b7c; - kbdev->regmap.regs[JOB_CONTROL__JS7__HEAD] = kbdev->reg + 0x1b80; - kbdev->regmap.regs[JOB_CONTROL__JS7__TAIL] = kbdev->reg + 0x1b88; - kbdev->regmap.regs[JOB_CONTROL__JS7__AFFINITY] = kbdev->reg + 0x1b90; - kbdev->regmap.regs[JOB_CONTROL__JS7__CONFIG] = kbdev->reg + 0x1b98; - kbdev->regmap.regs[JOB_CONTROL__JS7__XAFFINITY] = kbdev->reg + 0x1b9c; - kbdev->regmap.regs[JOB_CONTROL__JS7__COMMAND] = kbdev->reg + 0x1ba0; - kbdev->regmap.regs[JOB_CONTROL__JS7__STATUS] = kbdev->reg + 0x1ba4; - kbdev->regmap.regs[JOB_CONTROL__JS7__FLUSH_ID] = kbdev->reg + 0x1bb0; - kbdev->regmap.regs[JOB_CONTROL__JS7__EVENT_0] = kbdev->reg + 0x1bb4; - kbdev->regmap.regs[JOB_CONTROL__JS7__EVENT_1] = kbdev->reg + 0x1bb8; - kbdev->regmap.regs[JOB_CONTROL__JS7__EVENT_MASK] = kbdev->reg + 0x1bbc; - kbdev->regmap.regs[JOB_CONTROL__JS7__HEAD_NEXT] = kbdev->reg + 0x1bc0; - kbdev->regmap.regs[JOB_CONTROL__JS7__TAIL_NEXT] = kbdev->reg + 0x1bc8; - kbdev->regmap.regs[JOB_CONTROL__JS7__AFFINITY_NEXT] = kbdev->reg + 0x1bd0; - kbdev->regmap.regs[JOB_CONTROL__JS7__CONFIG_NEXT] = kbdev->reg + 0x1bd8; - kbdev->regmap.regs[JOB_CONTROL__JS7__XAFFINITY_NEXT] = kbdev->reg + 0x1bdc; - kbdev->regmap.regs[JOB_CONTROL__JS7__COMMAND_NEXT] = kbdev->reg + 0x1be0; - kbdev->regmap.regs[JOB_CONTROL__JS7__FLUSH_ID_NEXT] = kbdev->reg + 0x1bf0; - kbdev->regmap.regs[JOB_CONTROL__JS7__EVENT_MASK_NEXT] = kbdev->reg + 0x1bfc; - kbdev->regmap.regs[JOB_CONTROL__JS8__HEAD] = kbdev->reg + 0x1c00; - kbdev->regmap.regs[JOB_CONTROL__JS8__TAIL] = kbdev->reg + 0x1c08; - kbdev->regmap.regs[JOB_CONTROL__JS8__AFFINITY] = kbdev->reg + 0x1c10; - kbdev->regmap.regs[JOB_CONTROL__JS8__CONFIG] = kbdev->reg + 0x1c18; - kbdev->regmap.regs[JOB_CONTROL__JS8__XAFFINITY] = kbdev->reg + 0x1c1c; - kbdev->regmap.regs[JOB_CONTROL__JS8__COMMAND] = kbdev->reg + 0x1c20; - kbdev->regmap.regs[JOB_CONTROL__JS8__STATUS] = kbdev->reg + 0x1c24; - kbdev->regmap.regs[JOB_CONTROL__JS8__FLUSH_ID] = kbdev->reg + 0x1c30; - kbdev->regmap.regs[JOB_CONTROL__JS8__EVENT_0] = kbdev->reg + 0x1c34; - kbdev->regmap.regs[JOB_CONTROL__JS8__EVENT_1] = kbdev->reg + 0x1c38; - kbdev->regmap.regs[JOB_CONTROL__JS8__EVENT_MASK] = kbdev->reg + 0x1c3c; - kbdev->regmap.regs[JOB_CONTROL__JS8__HEAD_NEXT] = kbdev->reg + 0x1c40; - kbdev->regmap.regs[JOB_CONTROL__JS8__TAIL_NEXT] = kbdev->reg + 0x1c48; - kbdev->regmap.regs[JOB_CONTROL__JS8__AFFINITY_NEXT] = kbdev->reg + 0x1c50; - kbdev->regmap.regs[JOB_CONTROL__JS8__CONFIG_NEXT] = kbdev->reg + 0x1c58; - kbdev->regmap.regs[JOB_CONTROL__JS8__XAFFINITY_NEXT] = kbdev->reg + 0x1c5c; - kbdev->regmap.regs[JOB_CONTROL__JS8__COMMAND_NEXT] = kbdev->reg + 0x1c60; - kbdev->regmap.regs[JOB_CONTROL__JS8__FLUSH_ID_NEXT] = kbdev->reg + 0x1c70; - kbdev->regmap.regs[JOB_CONTROL__JS8__EVENT_MASK_NEXT] = kbdev->reg + 0x1c7c; - kbdev->regmap.regs[JOB_CONTROL__JS9__HEAD] = kbdev->reg + 0x1c80; - kbdev->regmap.regs[JOB_CONTROL__JS9__TAIL] = kbdev->reg + 0x1c88; - kbdev->regmap.regs[JOB_CONTROL__JS9__AFFINITY] = kbdev->reg + 0x1c90; - kbdev->regmap.regs[JOB_CONTROL__JS9__CONFIG] = kbdev->reg + 0x1c98; - kbdev->regmap.regs[JOB_CONTROL__JS9__XAFFINITY] = kbdev->reg + 0x1c9c; - kbdev->regmap.regs[JOB_CONTROL__JS9__COMMAND] = kbdev->reg + 0x1ca0; - kbdev->regmap.regs[JOB_CONTROL__JS9__STATUS] = kbdev->reg + 0x1ca4; - kbdev->regmap.regs[JOB_CONTROL__JS9__FLUSH_ID] = kbdev->reg + 0x1cb0; - kbdev->regmap.regs[JOB_CONTROL__JS9__EVENT_0] = kbdev->reg + 0x1cb4; - kbdev->regmap.regs[JOB_CONTROL__JS9__EVENT_1] = kbdev->reg + 0x1cb8; - kbdev->regmap.regs[JOB_CONTROL__JS9__EVENT_MASK] = kbdev->reg + 0x1cbc; - kbdev->regmap.regs[JOB_CONTROL__JS9__HEAD_NEXT] = kbdev->reg + 0x1cc0; - kbdev->regmap.regs[JOB_CONTROL__JS9__TAIL_NEXT] = kbdev->reg + 0x1cc8; - kbdev->regmap.regs[JOB_CONTROL__JS9__AFFINITY_NEXT] = kbdev->reg + 0x1cd0; - kbdev->regmap.regs[JOB_CONTROL__JS9__CONFIG_NEXT] = kbdev->reg + 0x1cd8; - kbdev->regmap.regs[JOB_CONTROL__JS9__XAFFINITY_NEXT] = kbdev->reg + 0x1cdc; - kbdev->regmap.regs[JOB_CONTROL__JS9__COMMAND_NEXT] = kbdev->reg + 0x1ce0; - kbdev->regmap.regs[JOB_CONTROL__JS9__FLUSH_ID_NEXT] = kbdev->reg + 0x1cf0; - kbdev->regmap.regs[JOB_CONTROL__JS9__EVENT_MASK_NEXT] = kbdev->reg + 0x1cfc; - kbdev->regmap.regs[JOB_CONTROL__JS10__HEAD] = kbdev->reg + 0x1d00; - kbdev->regmap.regs[JOB_CONTROL__JS10__TAIL] = kbdev->reg + 0x1d08; - kbdev->regmap.regs[JOB_CONTROL__JS10__AFFINITY] = kbdev->reg + 0x1d10; - kbdev->regmap.regs[JOB_CONTROL__JS10__CONFIG] = kbdev->reg + 0x1d18; - kbdev->regmap.regs[JOB_CONTROL__JS10__XAFFINITY] = kbdev->reg + 0x1d1c; - kbdev->regmap.regs[JOB_CONTROL__JS10__COMMAND] = kbdev->reg + 0x1d20; - kbdev->regmap.regs[JOB_CONTROL__JS10__STATUS] = kbdev->reg + 0x1d24; - kbdev->regmap.regs[JOB_CONTROL__JS10__FLUSH_ID] = kbdev->reg + 0x1d30; - kbdev->regmap.regs[JOB_CONTROL__JS10__EVENT_0] = kbdev->reg + 0x1d34; - kbdev->regmap.regs[JOB_CONTROL__JS10__EVENT_1] = kbdev->reg + 0x1d38; - kbdev->regmap.regs[JOB_CONTROL__JS10__EVENT_MASK] = kbdev->reg + 0x1d3c; - kbdev->regmap.regs[JOB_CONTROL__JS10__HEAD_NEXT] = kbdev->reg + 0x1d40; - kbdev->regmap.regs[JOB_CONTROL__JS10__TAIL_NEXT] = kbdev->reg + 0x1d48; - kbdev->regmap.regs[JOB_CONTROL__JS10__AFFINITY_NEXT] = kbdev->reg + 0x1d50; - kbdev->regmap.regs[JOB_CONTROL__JS10__CONFIG_NEXT] = kbdev->reg + 0x1d58; - kbdev->regmap.regs[JOB_CONTROL__JS10__XAFFINITY_NEXT] = kbdev->reg + 0x1d5c; - kbdev->regmap.regs[JOB_CONTROL__JS10__COMMAND_NEXT] = kbdev->reg + 0x1d60; - kbdev->regmap.regs[JOB_CONTROL__JS10__FLUSH_ID_NEXT] = kbdev->reg + 0x1d70; - kbdev->regmap.regs[JOB_CONTROL__JS10__EVENT_MASK_NEXT] = kbdev->reg + 0x1d7c; - kbdev->regmap.regs[JOB_CONTROL__JS11__HEAD] = kbdev->reg + 0x1d80; - kbdev->regmap.regs[JOB_CONTROL__JS11__TAIL] = kbdev->reg + 0x1d88; - kbdev->regmap.regs[JOB_CONTROL__JS11__AFFINITY] = kbdev->reg + 0x1d90; - kbdev->regmap.regs[JOB_CONTROL__JS11__CONFIG] = kbdev->reg + 0x1d98; - kbdev->regmap.regs[JOB_CONTROL__JS11__XAFFINITY] = kbdev->reg + 0x1d9c; - kbdev->regmap.regs[JOB_CONTROL__JS11__COMMAND] = kbdev->reg + 0x1da0; - kbdev->regmap.regs[JOB_CONTROL__JS11__STATUS] = kbdev->reg + 0x1da4; - kbdev->regmap.regs[JOB_CONTROL__JS11__FLUSH_ID] = kbdev->reg + 0x1db0; - kbdev->regmap.regs[JOB_CONTROL__JS11__EVENT_0] = kbdev->reg + 0x1db4; - kbdev->regmap.regs[JOB_CONTROL__JS11__EVENT_1] = kbdev->reg + 0x1db8; - kbdev->regmap.regs[JOB_CONTROL__JS11__EVENT_MASK] = kbdev->reg + 0x1dbc; - kbdev->regmap.regs[JOB_CONTROL__JS11__HEAD_NEXT] = kbdev->reg + 0x1dc0; - kbdev->regmap.regs[JOB_CONTROL__JS11__TAIL_NEXT] = kbdev->reg + 0x1dc8; - kbdev->regmap.regs[JOB_CONTROL__JS11__AFFINITY_NEXT] = kbdev->reg + 0x1dd0; - kbdev->regmap.regs[JOB_CONTROL__JS11__CONFIG_NEXT] = kbdev->reg + 0x1dd8; - kbdev->regmap.regs[JOB_CONTROL__JS11__XAFFINITY_NEXT] = kbdev->reg + 0x1ddc; - kbdev->regmap.regs[JOB_CONTROL__JS11__COMMAND_NEXT] = kbdev->reg + 0x1de0; - kbdev->regmap.regs[JOB_CONTROL__JS11__FLUSH_ID_NEXT] = kbdev->reg + 0x1df0; - kbdev->regmap.regs[JOB_CONTROL__JS11__EVENT_MASK_NEXT] = kbdev->reg + 0x1dfc; - kbdev->regmap.regs[JOB_CONTROL__JS12__HEAD] = kbdev->reg + 0x1e00; - kbdev->regmap.regs[JOB_CONTROL__JS12__TAIL] = kbdev->reg + 0x1e08; - kbdev->regmap.regs[JOB_CONTROL__JS12__AFFINITY] = kbdev->reg + 0x1e10; - kbdev->regmap.regs[JOB_CONTROL__JS12__CONFIG] = kbdev->reg + 0x1e18; - kbdev->regmap.regs[JOB_CONTROL__JS12__XAFFINITY] = kbdev->reg + 0x1e1c; - kbdev->regmap.regs[JOB_CONTROL__JS12__COMMAND] = kbdev->reg + 0x1e20; - kbdev->regmap.regs[JOB_CONTROL__JS12__STATUS] = kbdev->reg + 0x1e24; - kbdev->regmap.regs[JOB_CONTROL__JS12__FLUSH_ID] = kbdev->reg + 0x1e30; - kbdev->regmap.regs[JOB_CONTROL__JS12__EVENT_0] = kbdev->reg + 0x1e34; - kbdev->regmap.regs[JOB_CONTROL__JS12__EVENT_1] = kbdev->reg + 0x1e38; - kbdev->regmap.regs[JOB_CONTROL__JS12__EVENT_MASK] = kbdev->reg + 0x1e3c; - kbdev->regmap.regs[JOB_CONTROL__JS12__HEAD_NEXT] = kbdev->reg + 0x1e40; - kbdev->regmap.regs[JOB_CONTROL__JS12__TAIL_NEXT] = kbdev->reg + 0x1e48; - kbdev->regmap.regs[JOB_CONTROL__JS12__AFFINITY_NEXT] = kbdev->reg + 0x1e50; - kbdev->regmap.regs[JOB_CONTROL__JS12__CONFIG_NEXT] = kbdev->reg + 0x1e58; - kbdev->regmap.regs[JOB_CONTROL__JS12__XAFFINITY_NEXT] = kbdev->reg + 0x1e5c; - kbdev->regmap.regs[JOB_CONTROL__JS12__COMMAND_NEXT] = kbdev->reg + 0x1e60; - kbdev->regmap.regs[JOB_CONTROL__JS12__FLUSH_ID_NEXT] = kbdev->reg + 0x1e70; - kbdev->regmap.regs[JOB_CONTROL__JS12__EVENT_MASK_NEXT] = kbdev->reg + 0x1e7c; - kbdev->regmap.regs[JOB_CONTROL__JS13__HEAD] = kbdev->reg + 0x1e80; - kbdev->regmap.regs[JOB_CONTROL__JS13__TAIL] = kbdev->reg + 0x1e88; - kbdev->regmap.regs[JOB_CONTROL__JS13__AFFINITY] = kbdev->reg + 0x1e90; - kbdev->regmap.regs[JOB_CONTROL__JS13__CONFIG] = kbdev->reg + 0x1e98; - kbdev->regmap.regs[JOB_CONTROL__JS13__XAFFINITY] = kbdev->reg + 0x1e9c; - kbdev->regmap.regs[JOB_CONTROL__JS13__COMMAND] = kbdev->reg + 0x1ea0; - kbdev->regmap.regs[JOB_CONTROL__JS13__STATUS] = kbdev->reg + 0x1ea4; - kbdev->regmap.regs[JOB_CONTROL__JS13__FLUSH_ID] = kbdev->reg + 0x1eb0; - kbdev->regmap.regs[JOB_CONTROL__JS13__EVENT_0] = kbdev->reg + 0x1eb4; - kbdev->regmap.regs[JOB_CONTROL__JS13__EVENT_1] = kbdev->reg + 0x1eb8; - kbdev->regmap.regs[JOB_CONTROL__JS13__EVENT_MASK] = kbdev->reg + 0x1ebc; - kbdev->regmap.regs[JOB_CONTROL__JS13__HEAD_NEXT] = kbdev->reg + 0x1ec0; - kbdev->regmap.regs[JOB_CONTROL__JS13__TAIL_NEXT] = kbdev->reg + 0x1ec8; - kbdev->regmap.regs[JOB_CONTROL__JS13__AFFINITY_NEXT] = kbdev->reg + 0x1ed0; - kbdev->regmap.regs[JOB_CONTROL__JS13__CONFIG_NEXT] = kbdev->reg + 0x1ed8; - kbdev->regmap.regs[JOB_CONTROL__JS13__XAFFINITY_NEXT] = kbdev->reg + 0x1edc; - kbdev->regmap.regs[JOB_CONTROL__JS13__COMMAND_NEXT] = kbdev->reg + 0x1ee0; - kbdev->regmap.regs[JOB_CONTROL__JS13__FLUSH_ID_NEXT] = kbdev->reg + 0x1ef0; - kbdev->regmap.regs[JOB_CONTROL__JS13__EVENT_MASK_NEXT] = kbdev->reg + 0x1efc; - kbdev->regmap.regs[JOB_CONTROL__JS14__HEAD] = kbdev->reg + 0x1f00; - kbdev->regmap.regs[JOB_CONTROL__JS14__TAIL] = kbdev->reg + 0x1f08; - kbdev->regmap.regs[JOB_CONTROL__JS14__AFFINITY] = kbdev->reg + 0x1f10; - kbdev->regmap.regs[JOB_CONTROL__JS14__CONFIG] = kbdev->reg + 0x1f18; - kbdev->regmap.regs[JOB_CONTROL__JS14__XAFFINITY] = kbdev->reg + 0x1f1c; - kbdev->regmap.regs[JOB_CONTROL__JS14__COMMAND] = kbdev->reg + 0x1f20; - kbdev->regmap.regs[JOB_CONTROL__JS14__STATUS] = kbdev->reg + 0x1f24; - kbdev->regmap.regs[JOB_CONTROL__JS14__FLUSH_ID] = kbdev->reg + 0x1f30; - kbdev->regmap.regs[JOB_CONTROL__JS14__EVENT_0] = kbdev->reg + 0x1f34; - kbdev->regmap.regs[JOB_CONTROL__JS14__EVENT_1] = kbdev->reg + 0x1f38; - kbdev->regmap.regs[JOB_CONTROL__JS14__EVENT_MASK] = kbdev->reg + 0x1f3c; - kbdev->regmap.regs[JOB_CONTROL__JS14__HEAD_NEXT] = kbdev->reg + 0x1f40; - kbdev->regmap.regs[JOB_CONTROL__JS14__TAIL_NEXT] = kbdev->reg + 0x1f48; - kbdev->regmap.regs[JOB_CONTROL__JS14__AFFINITY_NEXT] = kbdev->reg + 0x1f50; - kbdev->regmap.regs[JOB_CONTROL__JS14__CONFIG_NEXT] = kbdev->reg + 0x1f58; - kbdev->regmap.regs[JOB_CONTROL__JS14__XAFFINITY_NEXT] = kbdev->reg + 0x1f5c; - kbdev->regmap.regs[JOB_CONTROL__JS14__COMMAND_NEXT] = kbdev->reg + 0x1f60; - kbdev->regmap.regs[JOB_CONTROL__JS14__FLUSH_ID_NEXT] = kbdev->reg + 0x1f70; - kbdev->regmap.regs[JOB_CONTROL__JS14__EVENT_MASK_NEXT] = kbdev->reg + 0x1f7c; - kbdev->regmap.regs[JOB_CONTROL__JS15__HEAD] = kbdev->reg + 0x1f80; - kbdev->regmap.regs[JOB_CONTROL__JS15__TAIL] = kbdev->reg + 0x1f88; - kbdev->regmap.regs[JOB_CONTROL__JS15__AFFINITY] = kbdev->reg + 0x1f90; - kbdev->regmap.regs[JOB_CONTROL__JS15__CONFIG] = kbdev->reg + 0x1f98; - kbdev->regmap.regs[JOB_CONTROL__JS15__XAFFINITY] = kbdev->reg + 0x1f9c; - kbdev->regmap.regs[JOB_CONTROL__JS15__COMMAND] = kbdev->reg + 0x1fa0; - kbdev->regmap.regs[JOB_CONTROL__JS15__STATUS] = kbdev->reg + 0x1fa4; - kbdev->regmap.regs[JOB_CONTROL__JS15__FLUSH_ID] = kbdev->reg + 0x1fb0; - kbdev->regmap.regs[JOB_CONTROL__JS15__EVENT_0] = kbdev->reg + 0x1fb4; - kbdev->regmap.regs[JOB_CONTROL__JS15__EVENT_1] = kbdev->reg + 0x1fb8; - kbdev->regmap.regs[JOB_CONTROL__JS15__EVENT_MASK] = kbdev->reg + 0x1fbc; - kbdev->regmap.regs[JOB_CONTROL__JS15__HEAD_NEXT] = kbdev->reg + 0x1fc0; - kbdev->regmap.regs[JOB_CONTROL__JS15__TAIL_NEXT] = kbdev->reg + 0x1fc8; - kbdev->regmap.regs[JOB_CONTROL__JS15__AFFINITY_NEXT] = kbdev->reg + 0x1fd0; - kbdev->regmap.regs[JOB_CONTROL__JS15__CONFIG_NEXT] = kbdev->reg + 0x1fd8; - kbdev->regmap.regs[JOB_CONTROL__JS15__XAFFINITY_NEXT] = kbdev->reg + 0x1fdc; - kbdev->regmap.regs[JOB_CONTROL__JS15__COMMAND_NEXT] = kbdev->reg + 0x1fe0; - kbdev->regmap.regs[JOB_CONTROL__JS15__FLUSH_ID_NEXT] = kbdev->reg + 0x1ff0; - kbdev->regmap.regs[JOB_CONTROL__JS15__EVENT_MASK_NEXT] = kbdev->reg + 0x1ffc; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__IRQ_RAWSTAT] = kbdev->reg + 0x2000; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__IRQ_CLEAR] = kbdev->reg + 0x2004; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__IRQ_MASK] = kbdev->reg + 0x2008; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__IRQ_STATUS] = kbdev->reg + 0x200c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__TRANSTAB] = kbdev->reg + 0x2400; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__MEMATTR] = kbdev->reg + 0x2408; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__LOCKADDR] = kbdev->reg + 0x2410; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__COMMAND] = kbdev->reg + 0x2418; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__FAULTSTATUS] = kbdev->reg + 0x241c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__FAULTADDRESS] = kbdev->reg + 0x2420; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__STATUS] = kbdev->reg + 0x2428; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__TRANSCFG] = kbdev->reg + 0x2430; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS0__FAULTEXTRA] = kbdev->reg + 0x2438; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__TRANSTAB] = kbdev->reg + 0x2440; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__MEMATTR] = kbdev->reg + 0x2448; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__LOCKADDR] = kbdev->reg + 0x2450; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__COMMAND] = kbdev->reg + 0x2458; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__FAULTSTATUS] = kbdev->reg + 0x245c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__FAULTADDRESS] = kbdev->reg + 0x2460; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__STATUS] = kbdev->reg + 0x2468; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__TRANSCFG] = kbdev->reg + 0x2470; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS1__FAULTEXTRA] = kbdev->reg + 0x2478; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__TRANSTAB] = kbdev->reg + 0x2480; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__MEMATTR] = kbdev->reg + 0x2488; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__LOCKADDR] = kbdev->reg + 0x2490; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__COMMAND] = kbdev->reg + 0x2498; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__FAULTSTATUS] = kbdev->reg + 0x249c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__FAULTADDRESS] = kbdev->reg + 0x24a0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__STATUS] = kbdev->reg + 0x24a8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__TRANSCFG] = kbdev->reg + 0x24b0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS2__FAULTEXTRA] = kbdev->reg + 0x24b8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__TRANSTAB] = kbdev->reg + 0x24c0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__MEMATTR] = kbdev->reg + 0x24c8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__LOCKADDR] = kbdev->reg + 0x24d0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__COMMAND] = kbdev->reg + 0x24d8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__FAULTSTATUS] = kbdev->reg + 0x24dc; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__FAULTADDRESS] = kbdev->reg + 0x24e0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__STATUS] = kbdev->reg + 0x24e8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__TRANSCFG] = kbdev->reg + 0x24f0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS3__FAULTEXTRA] = kbdev->reg + 0x24f8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__TRANSTAB] = kbdev->reg + 0x2500; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__MEMATTR] = kbdev->reg + 0x2508; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__LOCKADDR] = kbdev->reg + 0x2510; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__COMMAND] = kbdev->reg + 0x2518; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__FAULTSTATUS] = kbdev->reg + 0x251c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__FAULTADDRESS] = kbdev->reg + 0x2520; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__STATUS] = kbdev->reg + 0x2528; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__TRANSCFG] = kbdev->reg + 0x2530; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS4__FAULTEXTRA] = kbdev->reg + 0x2538; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__TRANSTAB] = kbdev->reg + 0x2540; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__MEMATTR] = kbdev->reg + 0x2548; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__LOCKADDR] = kbdev->reg + 0x2550; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__COMMAND] = kbdev->reg + 0x2558; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__FAULTSTATUS] = kbdev->reg + 0x255c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__FAULTADDRESS] = kbdev->reg + 0x2560; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__STATUS] = kbdev->reg + 0x2568; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__TRANSCFG] = kbdev->reg + 0x2570; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS5__FAULTEXTRA] = kbdev->reg + 0x2578; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__TRANSTAB] = kbdev->reg + 0x2580; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__MEMATTR] = kbdev->reg + 0x2588; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__LOCKADDR] = kbdev->reg + 0x2590; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__COMMAND] = kbdev->reg + 0x2598; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__FAULTSTATUS] = kbdev->reg + 0x259c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__FAULTADDRESS] = kbdev->reg + 0x25a0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__STATUS] = kbdev->reg + 0x25a8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__TRANSCFG] = kbdev->reg + 0x25b0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS6__FAULTEXTRA] = kbdev->reg + 0x25b8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__TRANSTAB] = kbdev->reg + 0x25c0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__MEMATTR] = kbdev->reg + 0x25c8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__LOCKADDR] = kbdev->reg + 0x25d0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__COMMAND] = kbdev->reg + 0x25d8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__FAULTSTATUS] = kbdev->reg + 0x25dc; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__FAULTADDRESS] = kbdev->reg + 0x25e0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__STATUS] = kbdev->reg + 0x25e8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__TRANSCFG] = kbdev->reg + 0x25f0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS7__FAULTEXTRA] = kbdev->reg + 0x25f8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__TRANSTAB] = kbdev->reg + 0x2600; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__MEMATTR] = kbdev->reg + 0x2608; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__LOCKADDR] = kbdev->reg + 0x2610; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__COMMAND] = kbdev->reg + 0x2618; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__FAULTSTATUS] = kbdev->reg + 0x261c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__FAULTADDRESS] = kbdev->reg + 0x2620; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__STATUS] = kbdev->reg + 0x2628; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__TRANSCFG] = kbdev->reg + 0x2630; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS8__FAULTEXTRA] = kbdev->reg + 0x2638; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__TRANSTAB] = kbdev->reg + 0x2640; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__MEMATTR] = kbdev->reg + 0x2648; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__LOCKADDR] = kbdev->reg + 0x2650; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__COMMAND] = kbdev->reg + 0x2658; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__FAULTSTATUS] = kbdev->reg + 0x265c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__FAULTADDRESS] = kbdev->reg + 0x2660; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__STATUS] = kbdev->reg + 0x2668; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__TRANSCFG] = kbdev->reg + 0x2670; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS9__FAULTEXTRA] = kbdev->reg + 0x2678; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__TRANSTAB] = kbdev->reg + 0x2680; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__MEMATTR] = kbdev->reg + 0x2688; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__LOCKADDR] = kbdev->reg + 0x2690; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__COMMAND] = kbdev->reg + 0x2698; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__FAULTSTATUS] = kbdev->reg + 0x269c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__FAULTADDRESS] = kbdev->reg + 0x26a0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__STATUS] = kbdev->reg + 0x26a8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__TRANSCFG] = kbdev->reg + 0x26b0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS10__FAULTEXTRA] = kbdev->reg + 0x26b8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__TRANSTAB] = kbdev->reg + 0x26c0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__MEMATTR] = kbdev->reg + 0x26c8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__LOCKADDR] = kbdev->reg + 0x26d0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__COMMAND] = kbdev->reg + 0x26d8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__FAULTSTATUS] = kbdev->reg + 0x26dc; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__FAULTADDRESS] = kbdev->reg + 0x26e0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__STATUS] = kbdev->reg + 0x26e8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__TRANSCFG] = kbdev->reg + 0x26f0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS11__FAULTEXTRA] = kbdev->reg + 0x26f8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__TRANSTAB] = kbdev->reg + 0x2700; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__MEMATTR] = kbdev->reg + 0x2708; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__LOCKADDR] = kbdev->reg + 0x2710; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__COMMAND] = kbdev->reg + 0x2718; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__FAULTSTATUS] = kbdev->reg + 0x271c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__FAULTADDRESS] = kbdev->reg + 0x2720; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__STATUS] = kbdev->reg + 0x2728; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__TRANSCFG] = kbdev->reg + 0x2730; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS12__FAULTEXTRA] = kbdev->reg + 0x2738; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__TRANSTAB] = kbdev->reg + 0x2740; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__MEMATTR] = kbdev->reg + 0x2748; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__LOCKADDR] = kbdev->reg + 0x2750; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__COMMAND] = kbdev->reg + 0x2758; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__FAULTSTATUS] = kbdev->reg + 0x275c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__FAULTADDRESS] = kbdev->reg + 0x2760; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__STATUS] = kbdev->reg + 0x2768; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__TRANSCFG] = kbdev->reg + 0x2770; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS13__FAULTEXTRA] = kbdev->reg + 0x2778; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__TRANSTAB] = kbdev->reg + 0x2780; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__MEMATTR] = kbdev->reg + 0x2788; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__LOCKADDR] = kbdev->reg + 0x2790; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__COMMAND] = kbdev->reg + 0x2798; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__FAULTSTATUS] = kbdev->reg + 0x279c; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__FAULTADDRESS] = kbdev->reg + 0x27a0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__STATUS] = kbdev->reg + 0x27a8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__TRANSCFG] = kbdev->reg + 0x27b0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS14__FAULTEXTRA] = kbdev->reg + 0x27b8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__TRANSTAB] = kbdev->reg + 0x27c0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__MEMATTR] = kbdev->reg + 0x27c8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__LOCKADDR] = kbdev->reg + 0x27d0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__COMMAND] = kbdev->reg + 0x27d8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__FAULTSTATUS] = kbdev->reg + 0x27dc; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__FAULTADDRESS] = kbdev->reg + 0x27e0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__STATUS] = kbdev->reg + 0x27e8; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__TRANSCFG] = kbdev->reg + 0x27f0; - kbdev->regmap.regs[MMU_STAGE1__ST1MMU__AS15__FAULTEXTRA] = kbdev->reg + 0x27f8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__IRQ_RAWSTAT] = kbdev->reg + 0x10000; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__IRQ_CLEAR] = kbdev->reg + 0x10004; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__IRQ_MASK] = kbdev->reg + 0x10008; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__IRQ_STATUS] = kbdev->reg + 0x1000c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__TRANSTAB] = kbdev->reg + 0x10400; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__MEMATTR] = kbdev->reg + 0x10408; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__LOCKADDR] = kbdev->reg + 0x10410; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__COMMAND] = kbdev->reg + 0x10418; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__FAULTSTATUS] = kbdev->reg + 0x1041c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__FAULTADDRESS] = kbdev->reg + 0x10420; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__STATUS] = kbdev->reg + 0x10428; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__TRANSCFG] = kbdev->reg + 0x10430; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS0__FAULTEXTRA] = kbdev->reg + 0x10438; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__TRANSTAB] = kbdev->reg + 0x10440; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__MEMATTR] = kbdev->reg + 0x10448; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__LOCKADDR] = kbdev->reg + 0x10450; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__COMMAND] = kbdev->reg + 0x10458; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__FAULTSTATUS] = kbdev->reg + 0x1045c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__FAULTADDRESS] = kbdev->reg + 0x10460; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__STATUS] = kbdev->reg + 0x10468; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__TRANSCFG] = kbdev->reg + 0x10470; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS1__FAULTEXTRA] = kbdev->reg + 0x10478; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__TRANSTAB] = kbdev->reg + 0x10480; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__MEMATTR] = kbdev->reg + 0x10488; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__LOCKADDR] = kbdev->reg + 0x10490; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__COMMAND] = kbdev->reg + 0x10498; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__FAULTSTATUS] = kbdev->reg + 0x1049c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__FAULTADDRESS] = kbdev->reg + 0x104a0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__STATUS] = kbdev->reg + 0x104a8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__TRANSCFG] = kbdev->reg + 0x104b0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS2__FAULTEXTRA] = kbdev->reg + 0x104b8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__TRANSTAB] = kbdev->reg + 0x104c0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__MEMATTR] = kbdev->reg + 0x104c8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__LOCKADDR] = kbdev->reg + 0x104d0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__COMMAND] = kbdev->reg + 0x104d8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__FAULTSTATUS] = kbdev->reg + 0x104dc; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__FAULTADDRESS] = kbdev->reg + 0x104e0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__STATUS] = kbdev->reg + 0x104e8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__TRANSCFG] = kbdev->reg + 0x104f0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS3__FAULTEXTRA] = kbdev->reg + 0x104f8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__TRANSTAB] = kbdev->reg + 0x10500; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__MEMATTR] = kbdev->reg + 0x10508; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__LOCKADDR] = kbdev->reg + 0x10510; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__COMMAND] = kbdev->reg + 0x10518; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__FAULTSTATUS] = kbdev->reg + 0x1051c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__FAULTADDRESS] = kbdev->reg + 0x10520; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__STATUS] = kbdev->reg + 0x10528; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__TRANSCFG] = kbdev->reg + 0x10530; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS4__FAULTEXTRA] = kbdev->reg + 0x10538; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__TRANSTAB] = kbdev->reg + 0x10540; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__MEMATTR] = kbdev->reg + 0x10548; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__LOCKADDR] = kbdev->reg + 0x10550; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__COMMAND] = kbdev->reg + 0x10558; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__FAULTSTATUS] = kbdev->reg + 0x1055c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__FAULTADDRESS] = kbdev->reg + 0x10560; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__STATUS] = kbdev->reg + 0x10568; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__TRANSCFG] = kbdev->reg + 0x10570; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS5__FAULTEXTRA] = kbdev->reg + 0x10578; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__TRANSTAB] = kbdev->reg + 0x10580; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__MEMATTR] = kbdev->reg + 0x10588; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__LOCKADDR] = kbdev->reg + 0x10590; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__COMMAND] = kbdev->reg + 0x10598; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__FAULTSTATUS] = kbdev->reg + 0x1059c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__FAULTADDRESS] = kbdev->reg + 0x105a0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__STATUS] = kbdev->reg + 0x105a8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__TRANSCFG] = kbdev->reg + 0x105b0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS6__FAULTEXTRA] = kbdev->reg + 0x105b8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__TRANSTAB] = kbdev->reg + 0x105c0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__MEMATTR] = kbdev->reg + 0x105c8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__LOCKADDR] = kbdev->reg + 0x105d0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__COMMAND] = kbdev->reg + 0x105d8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__FAULTSTATUS] = kbdev->reg + 0x105dc; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__FAULTADDRESS] = kbdev->reg + 0x105e0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__STATUS] = kbdev->reg + 0x105e8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__TRANSCFG] = kbdev->reg + 0x105f0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS7__FAULTEXTRA] = kbdev->reg + 0x105f8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__TRANSTAB] = kbdev->reg + 0x10600; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__MEMATTR] = kbdev->reg + 0x10608; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__LOCKADDR] = kbdev->reg + 0x10610; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__COMMAND] = kbdev->reg + 0x10618; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__FAULTSTATUS] = kbdev->reg + 0x1061c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__FAULTADDRESS] = kbdev->reg + 0x10620; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__STATUS] = kbdev->reg + 0x10628; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__TRANSCFG] = kbdev->reg + 0x10630; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS8__FAULTEXTRA] = kbdev->reg + 0x10638; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__TRANSTAB] = kbdev->reg + 0x10640; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__MEMATTR] = kbdev->reg + 0x10648; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__LOCKADDR] = kbdev->reg + 0x10650; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__COMMAND] = kbdev->reg + 0x10658; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__FAULTSTATUS] = kbdev->reg + 0x1065c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__FAULTADDRESS] = kbdev->reg + 0x10660; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__STATUS] = kbdev->reg + 0x10668; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__TRANSCFG] = kbdev->reg + 0x10670; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS9__FAULTEXTRA] = kbdev->reg + 0x10678; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__TRANSTAB] = kbdev->reg + 0x10680; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__MEMATTR] = kbdev->reg + 0x10688; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__LOCKADDR] = kbdev->reg + 0x10690; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__COMMAND] = kbdev->reg + 0x10698; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__FAULTSTATUS] = kbdev->reg + 0x1069c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__FAULTADDRESS] = kbdev->reg + 0x106a0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__STATUS] = kbdev->reg + 0x106a8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__TRANSCFG] = kbdev->reg + 0x106b0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS10__FAULTEXTRA] = kbdev->reg + 0x106b8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__TRANSTAB] = kbdev->reg + 0x106c0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__MEMATTR] = kbdev->reg + 0x106c8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__LOCKADDR] = kbdev->reg + 0x106d0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__COMMAND] = kbdev->reg + 0x106d8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__FAULTSTATUS] = kbdev->reg + 0x106dc; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__FAULTADDRESS] = kbdev->reg + 0x106e0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__STATUS] = kbdev->reg + 0x106e8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__TRANSCFG] = kbdev->reg + 0x106f0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS11__FAULTEXTRA] = kbdev->reg + 0x106f8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__TRANSTAB] = kbdev->reg + 0x10700; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__MEMATTR] = kbdev->reg + 0x10708; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__LOCKADDR] = kbdev->reg + 0x10710; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__COMMAND] = kbdev->reg + 0x10718; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__FAULTSTATUS] = kbdev->reg + 0x1071c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__FAULTADDRESS] = kbdev->reg + 0x10720; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__STATUS] = kbdev->reg + 0x10728; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__TRANSCFG] = kbdev->reg + 0x10730; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS12__FAULTEXTRA] = kbdev->reg + 0x10738; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__TRANSTAB] = kbdev->reg + 0x10740; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__MEMATTR] = kbdev->reg + 0x10748; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__LOCKADDR] = kbdev->reg + 0x10750; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__COMMAND] = kbdev->reg + 0x10758; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__FAULTSTATUS] = kbdev->reg + 0x1075c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__FAULTADDRESS] = kbdev->reg + 0x10760; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__STATUS] = kbdev->reg + 0x10768; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__TRANSCFG] = kbdev->reg + 0x10770; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS13__FAULTEXTRA] = kbdev->reg + 0x10778; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__TRANSTAB] = kbdev->reg + 0x10780; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__MEMATTR] = kbdev->reg + 0x10788; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__LOCKADDR] = kbdev->reg + 0x10790; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__COMMAND] = kbdev->reg + 0x10798; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__FAULTSTATUS] = kbdev->reg + 0x1079c; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__FAULTADDRESS] = kbdev->reg + 0x107a0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__STATUS] = kbdev->reg + 0x107a8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__TRANSCFG] = kbdev->reg + 0x107b0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS14__FAULTEXTRA] = kbdev->reg + 0x107b8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__TRANSTAB] = kbdev->reg + 0x107c0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__MEMATTR] = kbdev->reg + 0x107c8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__LOCKADDR] = kbdev->reg + 0x107d0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__COMMAND] = kbdev->reg + 0x107d8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__FAULTSTATUS] = kbdev->reg + 0x107dc; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__FAULTADDRESS] = kbdev->reg + 0x107e0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__STATUS] = kbdev->reg + 0x107e8; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__TRANSCFG] = kbdev->reg + 0x107f0; - kbdev->regmap.regs[MMU_STAGE2__ST2MMU__AS15__FAULTEXTRA] = kbdev->reg + 0x107f8; -} - -static void kbase_regmap_v6_2_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V6_2_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v6_0_init(kbdev); - - kbdev->regmap.flags[GPU_CONTROL__REVIDR] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__STACK_PRESENT] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__STACK_PWROFF] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__STACK_PWRON] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[GPU_CONTROL__STACK_PWRTRANS] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__STACK_READY] = KBASE_REGMAP_WIDTH_64_BIT | - KBASE_REGMAP_PERM_READ; - - kbdev->regmap.regs[GPU_CONTROL__REVIDR] = kbdev->reg + 0x280; - kbdev->regmap.regs[GPU_CONTROL__STACK_PRESENT] = kbdev->reg + 0xe00; - kbdev->regmap.regs[GPU_CONTROL__STACK_PWROFF] = kbdev->reg + 0xe30; - kbdev->regmap.regs[GPU_CONTROL__STACK_PWRON] = kbdev->reg + 0xe20; - kbdev->regmap.regs[GPU_CONTROL__STACK_PWRTRANS] = kbdev->reg + 0xe40; - kbdev->regmap.regs[GPU_CONTROL__STACK_READY] = kbdev->reg + 0xe10; -} - -static void kbase_regmap_v7_0_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V7_0_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v6_2_init(kbdev); - - kbdev->regmap.flags[GPU_CONTROL__TEXTURE_FEATURES_3] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - - kbdev->regmap.regs[GPU_CONTROL__TEXTURE_FEATURES_3] = kbdev->reg + 0xbc; -} - -static void kbase_regmap_v7_2_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V7_2_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v7_0_init(kbdev); - - kbdev->regmap.flags[GPU_CONTROL__CORE_FEATURES] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[GPU_CONTROL__THREAD_TLS_ALLOC] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - - kbdev->regmap.regs[GPU_CONTROL__CORE_FEATURES] = kbdev->reg + 0x8; - kbdev->regmap.regs[GPU_CONTROL__THREAD_TLS_ALLOC] = kbdev->reg + 0x310; -} - -static void kbase_regmap_v9_0_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V9_0_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v7_2_init(kbdev); - - kbdev->regmap.flags[GPU_CONTROL__CORE_FEATURES] = 0; - kbdev->regmap.flags[GPU_CONTROL__THREAD_TLS_ALLOC] = 0; - kbdev->regmap.flags[JOB_CONTROL__JOB_IRQ_THROTTLE] = 0; - - kbdev->regmap.regs[GPU_CONTROL__CORE_FEATURES] = NULL; - kbdev->regmap.regs[GPU_CONTROL__THREAD_TLS_ALLOC] = NULL; - kbdev->regmap.regs[JOB_CONTROL__JOB_IRQ_THROTTLE] = NULL; -} - -static void kbase_regmap_v9_2_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V9_2_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v9_0_init(kbdev); - - kbdev->regmap.flags[GPU_CONTROL__L2_CONFIG] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - - kbdev->regmap.regs[GPU_CONTROL__L2_CONFIG] = kbdev->reg + 0x48; -} - -static void kbase_regmap_v9_14_init(struct kbase_device *kbdev) -{ - if (kbdev->regmap.regs == NULL && kbdev->regmap.flags == NULL) { - kbdev->regmap.size = NR_V9_14_REGS; - kbdev->regmap.regs = - kcalloc(kbdev->regmap.size, sizeof(void __iomem *), GFP_KERNEL); - kbdev->regmap.flags = kcalloc(kbdev->regmap.size, sizeof(u32), GFP_KERNEL); - } - - if (WARN_ON(kbdev->regmap.regs == NULL)) - return; - if (WARN_ON(kbdev->regmap.flags == NULL)) - return; - - kbase_regmap_v9_2_init(kbdev); - - kbdev->regmap.flags[PTM_AW_IRQ_CLEAR] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[PTM_AW_IRQ_INJECTION] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[PTM_AW_IRQ_MASK] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | - KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[PTM_AW_IRQ_RAWSTAT] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[PTM_AW_IRQ_STATUS] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = KBASE_REGMAP_WIDTH_32_BIT | - KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ | KBASE_REGMAP_PERM_WRITE; - kbdev->regmap.flags[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] = - KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ; - kbdev->regmap.flags[PTM_ID] = KBASE_REGMAP_WIDTH_32_BIT | KBASE_REGMAP_PERM_READ; - - kbdev->regmap.regs[PTM_AW_IRQ_CLEAR] = kbdev->reg + 0x1ffc8; - kbdev->regmap.regs[PTM_AW_IRQ_INJECTION] = kbdev->reg + 0x1ffd4; - kbdev->regmap.regs[PTM_AW_IRQ_MASK] = kbdev->reg + 0x1ffcc; - kbdev->regmap.regs[PTM_AW_IRQ_RAWSTAT] = kbdev->reg + 0x1ffc4; - kbdev->regmap.regs[PTM_AW_IRQ_STATUS] = kbdev->reg + 0x1ffd0; - kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = kbdev->reg + 0x1ffd8; - kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = kbdev->reg + 0x1ffdc; - kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = kbdev->reg + 0x1ffe4; - kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = kbdev->reg + 0x1ffe8; - kbdev->regmap.regs[PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] = kbdev->reg + 0x1ffe0; - kbdev->regmap.regs[PTM_ID] = kbdev->reg + 0x1ffc0; -} - -u32 kbase_regmap_backend_init(struct kbase_device *kbdev) -{ - int i = 0; - - struct { - u32 arch_id; - void (*init)(struct kbase_device *kbdev); - } init_array[] = { - { GPU_ID_ARCH_MAKE(6, 0, 0), kbase_regmap_v6_0_init }, - { GPU_ID_ARCH_MAKE(6, 2, 0), kbase_regmap_v6_2_init }, - { GPU_ID_ARCH_MAKE(7, 0, 0), kbase_regmap_v7_0_init }, - { GPU_ID_ARCH_MAKE(7, 2, 0), kbase_regmap_v7_2_init }, - { GPU_ID_ARCH_MAKE(9, 0, 0), kbase_regmap_v9_0_init }, - { GPU_ID_ARCH_MAKE(9, 2, 0), kbase_regmap_v9_2_init }, - { GPU_ID_ARCH_MAKE(9, 14, 0), kbase_regmap_v9_14_init }, - }; - - for (i = 0; i < ARRAY_SIZE(init_array) - 1; i++) { - if (kbdev->gpu_props.gpu_id.arch_id < init_array[i + 1].arch_id) { - init_array[i].init(kbdev); - return init_array[i].arch_id; - } - } - - /* arch_id greater than last entry in init_array */ - init_array[i].init(kbdev); - return init_array[i].arch_id; -} - -#ifdef CONFIG_MALI_VALHALL_DEBUG -static char *enum_strings[] = { - [GPU_CONTROL__GPU_ID] = "GPU_CONTROL__GPU_ID", - [GPU_CONTROL__L2_FEATURES] = "GPU_CONTROL__L2_FEATURES", - [GPU_CONTROL__TILER_FEATURES] = "GPU_CONTROL__TILER_FEATURES", - [GPU_CONTROL__MEM_FEATURES] = "GPU_CONTROL__MEM_FEATURES", - [GPU_CONTROL__MMU_FEATURES] = "GPU_CONTROL__MMU_FEATURES", - [GPU_CONTROL__AS_PRESENT] = "GPU_CONTROL__AS_PRESENT", - [GPU_CONTROL__JS_PRESENT] = "GPU_CONTROL__JS_PRESENT", - [GPU_CONTROL__GPU_IRQ_RAWSTAT] = "GPU_CONTROL__GPU_IRQ_RAWSTAT", - [GPU_CONTROL__GPU_IRQ_CLEAR] = "GPU_CONTROL__GPU_IRQ_CLEAR", - [GPU_CONTROL__GPU_IRQ_MASK] = "GPU_CONTROL__GPU_IRQ_MASK", - [GPU_CONTROL__GPU_IRQ_STATUS] = "GPU_CONTROL__GPU_IRQ_STATUS", - [GPU_CONTROL__GPU_COMMAND] = "GPU_CONTROL__GPU_COMMAND", - [GPU_CONTROL__GPU_STATUS] = "GPU_CONTROL__GPU_STATUS", - [GPU_CONTROL__LATEST_FLUSH] = "GPU_CONTROL__LATEST_FLUSH", - [GPU_CONTROL__GPU_FAULTSTATUS] = "GPU_CONTROL__GPU_FAULTSTATUS", - [GPU_CONTROL__GPU_FAULTADDRESS] = "GPU_CONTROL__GPU_FAULTADDRESS", - [GPU_CONTROL__AFBC_FEATURES] = "GPU_CONTROL__AFBC_FEATURES", - [GPU_CONTROL__PWR_KEY] = "GPU_CONTROL__PWR_KEY", - [GPU_CONTROL__PWR_OVERRIDE0] = "GPU_CONTROL__PWR_OVERRIDE0", - [GPU_CONTROL__PWR_OVERRIDE1] = "GPU_CONTROL__PWR_OVERRIDE1", - [GPU_CONTROL__PRFCNT_BASE] = "GPU_CONTROL__PRFCNT_BASE", - [GPU_CONTROL__PRFCNT_CONFIG] = "GPU_CONTROL__PRFCNT_CONFIG", - [GPU_CONTROL__PRFCNT_JM_EN] = "GPU_CONTROL__PRFCNT_JM_EN", - [GPU_CONTROL__PRFCNT_SHADER_EN] = "GPU_CONTROL__PRFCNT_SHADER_EN", - [GPU_CONTROL__PRFCNT_TILER_EN] = "GPU_CONTROL__PRFCNT_TILER_EN", - [GPU_CONTROL__PRFCNT_MMU_L2_EN] = "GPU_CONTROL__PRFCNT_MMU_L2_EN", - [GPU_CONTROL__CYCLE_COUNT] = "GPU_CONTROL__CYCLE_COUNT", - [GPU_CONTROL__TIMESTAMP] = "GPU_CONTROL__TIMESTAMP", - [GPU_CONTROL__THREAD_MAX_THREADS] = "GPU_CONTROL__THREAD_MAX_THREADS", - [GPU_CONTROL__THREAD_MAX_WORKGROUP_SIZE] = "GPU_CONTROL__THREAD_MAX_WORKGROUP_SIZE", - [GPU_CONTROL__THREAD_MAX_BARRIER_SIZE] = "GPU_CONTROL__THREAD_MAX_BARRIER_SIZE", - [GPU_CONTROL__THREAD_FEATURES] = "GPU_CONTROL__THREAD_FEATURES", - [GPU_CONTROL__TEXTURE_FEATURES_0] = "GPU_CONTROL__TEXTURE_FEATURES_0", - [GPU_CONTROL__TEXTURE_FEATURES_1] = "GPU_CONTROL__TEXTURE_FEATURES_1", - [GPU_CONTROL__TEXTURE_FEATURES_2] = "GPU_CONTROL__TEXTURE_FEATURES_2", - [GPU_CONTROL__JS0_FEATURES] = "GPU_CONTROL__JS0_FEATURES", - [GPU_CONTROL__JS1_FEATURES] = "GPU_CONTROL__JS1_FEATURES", - [GPU_CONTROL__JS2_FEATURES] = "GPU_CONTROL__JS2_FEATURES", - [GPU_CONTROL__JS3_FEATURES] = "GPU_CONTROL__JS3_FEATURES", - [GPU_CONTROL__JS4_FEATURES] = "GPU_CONTROL__JS4_FEATURES", - [GPU_CONTROL__JS5_FEATURES] = "GPU_CONTROL__JS5_FEATURES", - [GPU_CONTROL__JS6_FEATURES] = "GPU_CONTROL__JS6_FEATURES", - [GPU_CONTROL__JS7_FEATURES] = "GPU_CONTROL__JS7_FEATURES", - [GPU_CONTROL__JS8_FEATURES] = "GPU_CONTROL__JS8_FEATURES", - [GPU_CONTROL__JS9_FEATURES] = "GPU_CONTROL__JS9_FEATURES", - [GPU_CONTROL__JS10_FEATURES] = "GPU_CONTROL__JS10_FEATURES", - [GPU_CONTROL__JS11_FEATURES] = "GPU_CONTROL__JS11_FEATURES", - [GPU_CONTROL__JS12_FEATURES] = "GPU_CONTROL__JS12_FEATURES", - [GPU_CONTROL__JS13_FEATURES] = "GPU_CONTROL__JS13_FEATURES", - [GPU_CONTROL__JS14_FEATURES] = "GPU_CONTROL__JS14_FEATURES", - [GPU_CONTROL__JS15_FEATURES] = "GPU_CONTROL__JS15_FEATURES", - [GPU_CONTROL__SHADER_PRESENT] = "GPU_CONTROL__SHADER_PRESENT", - [GPU_CONTROL__TILER_PRESENT] = "GPU_CONTROL__TILER_PRESENT", - [GPU_CONTROL__L2_PRESENT] = "GPU_CONTROL__L2_PRESENT", - [GPU_CONTROL__SHADER_READY] = "GPU_CONTROL__SHADER_READY", - [GPU_CONTROL__TILER_READY] = "GPU_CONTROL__TILER_READY", - [GPU_CONTROL__L2_READY] = "GPU_CONTROL__L2_READY", - [GPU_CONTROL__SHADER_PWRON] = "GPU_CONTROL__SHADER_PWRON", - [GPU_CONTROL__TILER_PWRON] = "GPU_CONTROL__TILER_PWRON", - [GPU_CONTROL__L2_PWRON] = "GPU_CONTROL__L2_PWRON", - [GPU_CONTROL__SHADER_PWROFF] = "GPU_CONTROL__SHADER_PWROFF", - [GPU_CONTROL__TILER_PWROFF] = "GPU_CONTROL__TILER_PWROFF", - [GPU_CONTROL__L2_PWROFF] = "GPU_CONTROL__L2_PWROFF", - [GPU_CONTROL__SHADER_PWRTRANS] = "GPU_CONTROL__SHADER_PWRTRANS", - [GPU_CONTROL__TILER_PWRTRANS] = "GPU_CONTROL__TILER_PWRTRANS", - [GPU_CONTROL__L2_PWRTRANS] = "GPU_CONTROL__L2_PWRTRANS", - [GPU_CONTROL__SHADER_PWRACTIVE] = "GPU_CONTROL__SHADER_PWRACTIVE", - [GPU_CONTROL__TILER_PWRACTIVE] = "GPU_CONTROL__TILER_PWRACTIVE", - [GPU_CONTROL__L2_PWRACTIVE] = "GPU_CONTROL__L2_PWRACTIVE", - [GPU_CONTROL__COHERENCY_FEATURES] = "GPU_CONTROL__COHERENCY_FEATURES", - [GPU_CONTROL__COHERENCY_ENABLE] = "GPU_CONTROL__COHERENCY_ENABLE", - [GPU_CONTROL__GPU_USER_INn] = "GPU_CONTROL__GPU_USER_INn", - [GPU_CONTROL__GPU_USER_OUTn] = "GPU_CONTROL__GPU_USER_OUTn", - [GPU_CONTROL__JM_CONFIG] = "GPU_CONTROL__JM_CONFIG", - [GPU_CONTROL__SHADER_CONFIG] = "GPU_CONTROL__SHADER_CONFIG", - [GPU_CONTROL__TILER_CONFIG] = "GPU_CONTROL__TILER_CONFIG", - [GPU_CONTROL__L2_MMU_CONFIG] = "GPU_CONTROL__L2_MMU_CONFIG", - [JOB_CONTROL__JOB_IRQ_RAWSTAT] = "JOB_CONTROL__JOB_IRQ_RAWSTAT", - [JOB_CONTROL__JOB_IRQ_CLEAR] = "JOB_CONTROL__JOB_IRQ_CLEAR", - [JOB_CONTROL__JOB_IRQ_MASK] = "JOB_CONTROL__JOB_IRQ_MASK", - [JOB_CONTROL__JOB_IRQ_STATUS] = "JOB_CONTROL__JOB_IRQ_STATUS", - [JOB_CONTROL__JOB_IRQ_JS_STATE] = "JOB_CONTROL__JOB_IRQ_JS_STATE", - [JOB_CONTROL__JOB_IRQ_THROTTLE] = "JOB_CONTROL__JOB_IRQ_THROTTLE", - [JOB_CONTROL__EVENT_IRQ_RAWSTAT] = "JOB_CONTROL__EVENT_IRQ_RAWSTAT", - [JOB_CONTROL__EVENT_IRQ_CLEAR] = "JOB_CONTROL__EVENT_IRQ_CLEAR", - [JOB_CONTROL__EVENT_IRQ_MASK] = "JOB_CONTROL__EVENT_IRQ_MASK", - [JOB_CONTROL__EVENT_IRQ_STATUS] = "JOB_CONTROL__EVENT_IRQ_STATUS", - [JOB_CONTROL__JS0__HEAD] = "JOB_CONTROL__JS0__HEAD", - [JOB_CONTROL__JS0__TAIL] = "JOB_CONTROL__JS0__TAIL", - [JOB_CONTROL__JS0__AFFINITY] = "JOB_CONTROL__JS0__AFFINITY", - [JOB_CONTROL__JS0__CONFIG] = "JOB_CONTROL__JS0__CONFIG", - [JOB_CONTROL__JS0__XAFFINITY] = "JOB_CONTROL__JS0__XAFFINITY", - [JOB_CONTROL__JS0__COMMAND] = "JOB_CONTROL__JS0__COMMAND", - [JOB_CONTROL__JS0__STATUS] = "JOB_CONTROL__JS0__STATUS", - [JOB_CONTROL__JS0__FLUSH_ID] = "JOB_CONTROL__JS0__FLUSH_ID", - [JOB_CONTROL__JS0__EVENT_0] = "JOB_CONTROL__JS0__EVENT_0", - [JOB_CONTROL__JS0__EVENT_1] = "JOB_CONTROL__JS0__EVENT_1", - [JOB_CONTROL__JS0__EVENT_MASK] = "JOB_CONTROL__JS0__EVENT_MASK", - [JOB_CONTROL__JS0__HEAD_NEXT] = "JOB_CONTROL__JS0__HEAD_NEXT", - [JOB_CONTROL__JS0__TAIL_NEXT] = "JOB_CONTROL__JS0__TAIL_NEXT", - [JOB_CONTROL__JS0__AFFINITY_NEXT] = "JOB_CONTROL__JS0__AFFINITY_NEXT", - [JOB_CONTROL__JS0__CONFIG_NEXT] = "JOB_CONTROL__JS0__CONFIG_NEXT", - [JOB_CONTROL__JS0__XAFFINITY_NEXT] = "JOB_CONTROL__JS0__XAFFINITY_NEXT", - [JOB_CONTROL__JS0__COMMAND_NEXT] = "JOB_CONTROL__JS0__COMMAND_NEXT", - [JOB_CONTROL__JS0__FLUSH_ID_NEXT] = "JOB_CONTROL__JS0__FLUSH_ID_NEXT", - [JOB_CONTROL__JS0__EVENT_MASK_NEXT] = "JOB_CONTROL__JS0__EVENT_MASK_NEXT", - [JOB_CONTROL__JS1__HEAD] = "JOB_CONTROL__JS1__HEAD", - [JOB_CONTROL__JS1__TAIL] = "JOB_CONTROL__JS1__TAIL", - [JOB_CONTROL__JS1__AFFINITY] = "JOB_CONTROL__JS1__AFFINITY", - [JOB_CONTROL__JS1__CONFIG] = "JOB_CONTROL__JS1__CONFIG", - [JOB_CONTROL__JS1__XAFFINITY] = "JOB_CONTROL__JS1__XAFFINITY", - [JOB_CONTROL__JS1__COMMAND] = "JOB_CONTROL__JS1__COMMAND", - [JOB_CONTROL__JS1__STATUS] = "JOB_CONTROL__JS1__STATUS", - [JOB_CONTROL__JS1__FLUSH_ID] = "JOB_CONTROL__JS1__FLUSH_ID", - [JOB_CONTROL__JS1__EVENT_0] = "JOB_CONTROL__JS1__EVENT_0", - [JOB_CONTROL__JS1__EVENT_1] = "JOB_CONTROL__JS1__EVENT_1", - [JOB_CONTROL__JS1__EVENT_MASK] = "JOB_CONTROL__JS1__EVENT_MASK", - [JOB_CONTROL__JS1__HEAD_NEXT] = "JOB_CONTROL__JS1__HEAD_NEXT", - [JOB_CONTROL__JS1__TAIL_NEXT] = "JOB_CONTROL__JS1__TAIL_NEXT", - [JOB_CONTROL__JS1__AFFINITY_NEXT] = "JOB_CONTROL__JS1__AFFINITY_NEXT", - [JOB_CONTROL__JS1__CONFIG_NEXT] = "JOB_CONTROL__JS1__CONFIG_NEXT", - [JOB_CONTROL__JS1__XAFFINITY_NEXT] = "JOB_CONTROL__JS1__XAFFINITY_NEXT", - [JOB_CONTROL__JS1__COMMAND_NEXT] = "JOB_CONTROL__JS1__COMMAND_NEXT", - [JOB_CONTROL__JS1__FLUSH_ID_NEXT] = "JOB_CONTROL__JS1__FLUSH_ID_NEXT", - [JOB_CONTROL__JS1__EVENT_MASK_NEXT] = "JOB_CONTROL__JS1__EVENT_MASK_NEXT", - [JOB_CONTROL__JS2__HEAD] = "JOB_CONTROL__JS2__HEAD", - [JOB_CONTROL__JS2__TAIL] = "JOB_CONTROL__JS2__TAIL", - [JOB_CONTROL__JS2__AFFINITY] = "JOB_CONTROL__JS2__AFFINITY", - [JOB_CONTROL__JS2__CONFIG] = "JOB_CONTROL__JS2__CONFIG", - [JOB_CONTROL__JS2__XAFFINITY] = "JOB_CONTROL__JS2__XAFFINITY", - [JOB_CONTROL__JS2__COMMAND] = "JOB_CONTROL__JS2__COMMAND", - [JOB_CONTROL__JS2__STATUS] = "JOB_CONTROL__JS2__STATUS", - [JOB_CONTROL__JS2__FLUSH_ID] = "JOB_CONTROL__JS2__FLUSH_ID", - [JOB_CONTROL__JS2__EVENT_0] = "JOB_CONTROL__JS2__EVENT_0", - [JOB_CONTROL__JS2__EVENT_1] = "JOB_CONTROL__JS2__EVENT_1", - [JOB_CONTROL__JS2__EVENT_MASK] = "JOB_CONTROL__JS2__EVENT_MASK", - [JOB_CONTROL__JS2__HEAD_NEXT] = "JOB_CONTROL__JS2__HEAD_NEXT", - [JOB_CONTROL__JS2__TAIL_NEXT] = "JOB_CONTROL__JS2__TAIL_NEXT", - [JOB_CONTROL__JS2__AFFINITY_NEXT] = "JOB_CONTROL__JS2__AFFINITY_NEXT", - [JOB_CONTROL__JS2__CONFIG_NEXT] = "JOB_CONTROL__JS2__CONFIG_NEXT", - [JOB_CONTROL__JS2__XAFFINITY_NEXT] = "JOB_CONTROL__JS2__XAFFINITY_NEXT", - [JOB_CONTROL__JS2__COMMAND_NEXT] = "JOB_CONTROL__JS2__COMMAND_NEXT", - [JOB_CONTROL__JS2__FLUSH_ID_NEXT] = "JOB_CONTROL__JS2__FLUSH_ID_NEXT", - [JOB_CONTROL__JS2__EVENT_MASK_NEXT] = "JOB_CONTROL__JS2__EVENT_MASK_NEXT", - [JOB_CONTROL__JS3__HEAD] = "JOB_CONTROL__JS3__HEAD", - [JOB_CONTROL__JS3__TAIL] = "JOB_CONTROL__JS3__TAIL", - [JOB_CONTROL__JS3__AFFINITY] = "JOB_CONTROL__JS3__AFFINITY", - [JOB_CONTROL__JS3__CONFIG] = "JOB_CONTROL__JS3__CONFIG", - [JOB_CONTROL__JS3__XAFFINITY] = "JOB_CONTROL__JS3__XAFFINITY", - [JOB_CONTROL__JS3__COMMAND] = "JOB_CONTROL__JS3__COMMAND", - [JOB_CONTROL__JS3__STATUS] = "JOB_CONTROL__JS3__STATUS", - [JOB_CONTROL__JS3__FLUSH_ID] = "JOB_CONTROL__JS3__FLUSH_ID", - [JOB_CONTROL__JS3__EVENT_0] = "JOB_CONTROL__JS3__EVENT_0", - [JOB_CONTROL__JS3__EVENT_1] = "JOB_CONTROL__JS3__EVENT_1", - [JOB_CONTROL__JS3__EVENT_MASK] = "JOB_CONTROL__JS3__EVENT_MASK", - [JOB_CONTROL__JS3__HEAD_NEXT] = "JOB_CONTROL__JS3__HEAD_NEXT", - [JOB_CONTROL__JS3__TAIL_NEXT] = "JOB_CONTROL__JS3__TAIL_NEXT", - [JOB_CONTROL__JS3__AFFINITY_NEXT] = "JOB_CONTROL__JS3__AFFINITY_NEXT", - [JOB_CONTROL__JS3__CONFIG_NEXT] = "JOB_CONTROL__JS3__CONFIG_NEXT", - [JOB_CONTROL__JS3__XAFFINITY_NEXT] = "JOB_CONTROL__JS3__XAFFINITY_NEXT", - [JOB_CONTROL__JS3__COMMAND_NEXT] = "JOB_CONTROL__JS3__COMMAND_NEXT", - [JOB_CONTROL__JS3__FLUSH_ID_NEXT] = "JOB_CONTROL__JS3__FLUSH_ID_NEXT", - [JOB_CONTROL__JS3__EVENT_MASK_NEXT] = "JOB_CONTROL__JS3__EVENT_MASK_NEXT", - [JOB_CONTROL__JS4__HEAD] = "JOB_CONTROL__JS4__HEAD", - [JOB_CONTROL__JS4__TAIL] = "JOB_CONTROL__JS4__TAIL", - [JOB_CONTROL__JS4__AFFINITY] = "JOB_CONTROL__JS4__AFFINITY", - [JOB_CONTROL__JS4__CONFIG] = "JOB_CONTROL__JS4__CONFIG", - [JOB_CONTROL__JS4__XAFFINITY] = "JOB_CONTROL__JS4__XAFFINITY", - [JOB_CONTROL__JS4__COMMAND] = "JOB_CONTROL__JS4__COMMAND", - [JOB_CONTROL__JS4__STATUS] = "JOB_CONTROL__JS4__STATUS", - [JOB_CONTROL__JS4__FLUSH_ID] = "JOB_CONTROL__JS4__FLUSH_ID", - [JOB_CONTROL__JS4__EVENT_0] = "JOB_CONTROL__JS4__EVENT_0", - [JOB_CONTROL__JS4__EVENT_1] = "JOB_CONTROL__JS4__EVENT_1", - [JOB_CONTROL__JS4__EVENT_MASK] = "JOB_CONTROL__JS4__EVENT_MASK", - [JOB_CONTROL__JS4__HEAD_NEXT] = "JOB_CONTROL__JS4__HEAD_NEXT", - [JOB_CONTROL__JS4__TAIL_NEXT] = "JOB_CONTROL__JS4__TAIL_NEXT", - [JOB_CONTROL__JS4__AFFINITY_NEXT] = "JOB_CONTROL__JS4__AFFINITY_NEXT", - [JOB_CONTROL__JS4__CONFIG_NEXT] = "JOB_CONTROL__JS4__CONFIG_NEXT", - [JOB_CONTROL__JS4__XAFFINITY_NEXT] = "JOB_CONTROL__JS4__XAFFINITY_NEXT", - [JOB_CONTROL__JS4__COMMAND_NEXT] = "JOB_CONTROL__JS4__COMMAND_NEXT", - [JOB_CONTROL__JS4__FLUSH_ID_NEXT] = "JOB_CONTROL__JS4__FLUSH_ID_NEXT", - [JOB_CONTROL__JS4__EVENT_MASK_NEXT] = "JOB_CONTROL__JS4__EVENT_MASK_NEXT", - [JOB_CONTROL__JS5__HEAD] = "JOB_CONTROL__JS5__HEAD", - [JOB_CONTROL__JS5__TAIL] = "JOB_CONTROL__JS5__TAIL", - [JOB_CONTROL__JS5__AFFINITY] = "JOB_CONTROL__JS5__AFFINITY", - [JOB_CONTROL__JS5__CONFIG] = "JOB_CONTROL__JS5__CONFIG", - [JOB_CONTROL__JS5__XAFFINITY] = "JOB_CONTROL__JS5__XAFFINITY", - [JOB_CONTROL__JS5__COMMAND] = "JOB_CONTROL__JS5__COMMAND", - [JOB_CONTROL__JS5__STATUS] = "JOB_CONTROL__JS5__STATUS", - [JOB_CONTROL__JS5__FLUSH_ID] = "JOB_CONTROL__JS5__FLUSH_ID", - [JOB_CONTROL__JS5__EVENT_0] = "JOB_CONTROL__JS5__EVENT_0", - [JOB_CONTROL__JS5__EVENT_1] = "JOB_CONTROL__JS5__EVENT_1", - [JOB_CONTROL__JS5__EVENT_MASK] = "JOB_CONTROL__JS5__EVENT_MASK", - [JOB_CONTROL__JS5__HEAD_NEXT] = "JOB_CONTROL__JS5__HEAD_NEXT", - [JOB_CONTROL__JS5__TAIL_NEXT] = "JOB_CONTROL__JS5__TAIL_NEXT", - [JOB_CONTROL__JS5__AFFINITY_NEXT] = "JOB_CONTROL__JS5__AFFINITY_NEXT", - [JOB_CONTROL__JS5__CONFIG_NEXT] = "JOB_CONTROL__JS5__CONFIG_NEXT", - [JOB_CONTROL__JS5__XAFFINITY_NEXT] = "JOB_CONTROL__JS5__XAFFINITY_NEXT", - [JOB_CONTROL__JS5__COMMAND_NEXT] = "JOB_CONTROL__JS5__COMMAND_NEXT", - [JOB_CONTROL__JS5__FLUSH_ID_NEXT] = "JOB_CONTROL__JS5__FLUSH_ID_NEXT", - [JOB_CONTROL__JS5__EVENT_MASK_NEXT] = "JOB_CONTROL__JS5__EVENT_MASK_NEXT", - [JOB_CONTROL__JS6__HEAD] = "JOB_CONTROL__JS6__HEAD", - [JOB_CONTROL__JS6__TAIL] = "JOB_CONTROL__JS6__TAIL", - [JOB_CONTROL__JS6__AFFINITY] = "JOB_CONTROL__JS6__AFFINITY", - [JOB_CONTROL__JS6__CONFIG] = "JOB_CONTROL__JS6__CONFIG", - [JOB_CONTROL__JS6__XAFFINITY] = "JOB_CONTROL__JS6__XAFFINITY", - [JOB_CONTROL__JS6__COMMAND] = "JOB_CONTROL__JS6__COMMAND", - [JOB_CONTROL__JS6__STATUS] = "JOB_CONTROL__JS6__STATUS", - [JOB_CONTROL__JS6__FLUSH_ID] = "JOB_CONTROL__JS6__FLUSH_ID", - [JOB_CONTROL__JS6__EVENT_0] = "JOB_CONTROL__JS6__EVENT_0", - [JOB_CONTROL__JS6__EVENT_1] = "JOB_CONTROL__JS6__EVENT_1", - [JOB_CONTROL__JS6__EVENT_MASK] = "JOB_CONTROL__JS6__EVENT_MASK", - [JOB_CONTROL__JS6__HEAD_NEXT] = "JOB_CONTROL__JS6__HEAD_NEXT", - [JOB_CONTROL__JS6__TAIL_NEXT] = "JOB_CONTROL__JS6__TAIL_NEXT", - [JOB_CONTROL__JS6__AFFINITY_NEXT] = "JOB_CONTROL__JS6__AFFINITY_NEXT", - [JOB_CONTROL__JS6__CONFIG_NEXT] = "JOB_CONTROL__JS6__CONFIG_NEXT", - [JOB_CONTROL__JS6__XAFFINITY_NEXT] = "JOB_CONTROL__JS6__XAFFINITY_NEXT", - [JOB_CONTROL__JS6__COMMAND_NEXT] = "JOB_CONTROL__JS6__COMMAND_NEXT", - [JOB_CONTROL__JS6__FLUSH_ID_NEXT] = "JOB_CONTROL__JS6__FLUSH_ID_NEXT", - [JOB_CONTROL__JS6__EVENT_MASK_NEXT] = "JOB_CONTROL__JS6__EVENT_MASK_NEXT", - [JOB_CONTROL__JS7__HEAD] = "JOB_CONTROL__JS7__HEAD", - [JOB_CONTROL__JS7__TAIL] = "JOB_CONTROL__JS7__TAIL", - [JOB_CONTROL__JS7__AFFINITY] = "JOB_CONTROL__JS7__AFFINITY", - [JOB_CONTROL__JS7__CONFIG] = "JOB_CONTROL__JS7__CONFIG", - [JOB_CONTROL__JS7__XAFFINITY] = "JOB_CONTROL__JS7__XAFFINITY", - [JOB_CONTROL__JS7__COMMAND] = "JOB_CONTROL__JS7__COMMAND", - [JOB_CONTROL__JS7__STATUS] = "JOB_CONTROL__JS7__STATUS", - [JOB_CONTROL__JS7__FLUSH_ID] = "JOB_CONTROL__JS7__FLUSH_ID", - [JOB_CONTROL__JS7__EVENT_0] = "JOB_CONTROL__JS7__EVENT_0", - [JOB_CONTROL__JS7__EVENT_1] = "JOB_CONTROL__JS7__EVENT_1", - [JOB_CONTROL__JS7__EVENT_MASK] = "JOB_CONTROL__JS7__EVENT_MASK", - [JOB_CONTROL__JS7__HEAD_NEXT] = "JOB_CONTROL__JS7__HEAD_NEXT", - [JOB_CONTROL__JS7__TAIL_NEXT] = "JOB_CONTROL__JS7__TAIL_NEXT", - [JOB_CONTROL__JS7__AFFINITY_NEXT] = "JOB_CONTROL__JS7__AFFINITY_NEXT", - [JOB_CONTROL__JS7__CONFIG_NEXT] = "JOB_CONTROL__JS7__CONFIG_NEXT", - [JOB_CONTROL__JS7__XAFFINITY_NEXT] = "JOB_CONTROL__JS7__XAFFINITY_NEXT", - [JOB_CONTROL__JS7__COMMAND_NEXT] = "JOB_CONTROL__JS7__COMMAND_NEXT", - [JOB_CONTROL__JS7__FLUSH_ID_NEXT] = "JOB_CONTROL__JS7__FLUSH_ID_NEXT", - [JOB_CONTROL__JS7__EVENT_MASK_NEXT] = "JOB_CONTROL__JS7__EVENT_MASK_NEXT", - [JOB_CONTROL__JS8__HEAD] = "JOB_CONTROL__JS8__HEAD", - [JOB_CONTROL__JS8__TAIL] = "JOB_CONTROL__JS8__TAIL", - [JOB_CONTROL__JS8__AFFINITY] = "JOB_CONTROL__JS8__AFFINITY", - [JOB_CONTROL__JS8__CONFIG] = "JOB_CONTROL__JS8__CONFIG", - [JOB_CONTROL__JS8__XAFFINITY] = "JOB_CONTROL__JS8__XAFFINITY", - [JOB_CONTROL__JS8__COMMAND] = "JOB_CONTROL__JS8__COMMAND", - [JOB_CONTROL__JS8__STATUS] = "JOB_CONTROL__JS8__STATUS", - [JOB_CONTROL__JS8__FLUSH_ID] = "JOB_CONTROL__JS8__FLUSH_ID", - [JOB_CONTROL__JS8__EVENT_0] = "JOB_CONTROL__JS8__EVENT_0", - [JOB_CONTROL__JS8__EVENT_1] = "JOB_CONTROL__JS8__EVENT_1", - [JOB_CONTROL__JS8__EVENT_MASK] = "JOB_CONTROL__JS8__EVENT_MASK", - [JOB_CONTROL__JS8__HEAD_NEXT] = "JOB_CONTROL__JS8__HEAD_NEXT", - [JOB_CONTROL__JS8__TAIL_NEXT] = "JOB_CONTROL__JS8__TAIL_NEXT", - [JOB_CONTROL__JS8__AFFINITY_NEXT] = "JOB_CONTROL__JS8__AFFINITY_NEXT", - [JOB_CONTROL__JS8__CONFIG_NEXT] = "JOB_CONTROL__JS8__CONFIG_NEXT", - [JOB_CONTROL__JS8__XAFFINITY_NEXT] = "JOB_CONTROL__JS8__XAFFINITY_NEXT", - [JOB_CONTROL__JS8__COMMAND_NEXT] = "JOB_CONTROL__JS8__COMMAND_NEXT", - [JOB_CONTROL__JS8__FLUSH_ID_NEXT] = "JOB_CONTROL__JS8__FLUSH_ID_NEXT", - [JOB_CONTROL__JS8__EVENT_MASK_NEXT] = "JOB_CONTROL__JS8__EVENT_MASK_NEXT", - [JOB_CONTROL__JS9__HEAD] = "JOB_CONTROL__JS9__HEAD", - [JOB_CONTROL__JS9__TAIL] = "JOB_CONTROL__JS9__TAIL", - [JOB_CONTROL__JS9__AFFINITY] = "JOB_CONTROL__JS9__AFFINITY", - [JOB_CONTROL__JS9__CONFIG] = "JOB_CONTROL__JS9__CONFIG", - [JOB_CONTROL__JS9__XAFFINITY] = "JOB_CONTROL__JS9__XAFFINITY", - [JOB_CONTROL__JS9__COMMAND] = "JOB_CONTROL__JS9__COMMAND", - [JOB_CONTROL__JS9__STATUS] = "JOB_CONTROL__JS9__STATUS", - [JOB_CONTROL__JS9__FLUSH_ID] = "JOB_CONTROL__JS9__FLUSH_ID", - [JOB_CONTROL__JS9__EVENT_0] = "JOB_CONTROL__JS9__EVENT_0", - [JOB_CONTROL__JS9__EVENT_1] = "JOB_CONTROL__JS9__EVENT_1", - [JOB_CONTROL__JS9__EVENT_MASK] = "JOB_CONTROL__JS9__EVENT_MASK", - [JOB_CONTROL__JS9__HEAD_NEXT] = "JOB_CONTROL__JS9__HEAD_NEXT", - [JOB_CONTROL__JS9__TAIL_NEXT] = "JOB_CONTROL__JS9__TAIL_NEXT", - [JOB_CONTROL__JS9__AFFINITY_NEXT] = "JOB_CONTROL__JS9__AFFINITY_NEXT", - [JOB_CONTROL__JS9__CONFIG_NEXT] = "JOB_CONTROL__JS9__CONFIG_NEXT", - [JOB_CONTROL__JS9__XAFFINITY_NEXT] = "JOB_CONTROL__JS9__XAFFINITY_NEXT", - [JOB_CONTROL__JS9__COMMAND_NEXT] = "JOB_CONTROL__JS9__COMMAND_NEXT", - [JOB_CONTROL__JS9__FLUSH_ID_NEXT] = "JOB_CONTROL__JS9__FLUSH_ID_NEXT", - [JOB_CONTROL__JS9__EVENT_MASK_NEXT] = "JOB_CONTROL__JS9__EVENT_MASK_NEXT", - [JOB_CONTROL__JS10__HEAD] = "JOB_CONTROL__JS10__HEAD", - [JOB_CONTROL__JS10__TAIL] = "JOB_CONTROL__JS10__TAIL", - [JOB_CONTROL__JS10__AFFINITY] = "JOB_CONTROL__JS10__AFFINITY", - [JOB_CONTROL__JS10__CONFIG] = "JOB_CONTROL__JS10__CONFIG", - [JOB_CONTROL__JS10__XAFFINITY] = "JOB_CONTROL__JS10__XAFFINITY", - [JOB_CONTROL__JS10__COMMAND] = "JOB_CONTROL__JS10__COMMAND", - [JOB_CONTROL__JS10__STATUS] = "JOB_CONTROL__JS10__STATUS", - [JOB_CONTROL__JS10__FLUSH_ID] = "JOB_CONTROL__JS10__FLUSH_ID", - [JOB_CONTROL__JS10__EVENT_0] = "JOB_CONTROL__JS10__EVENT_0", - [JOB_CONTROL__JS10__EVENT_1] = "JOB_CONTROL__JS10__EVENT_1", - [JOB_CONTROL__JS10__EVENT_MASK] = "JOB_CONTROL__JS10__EVENT_MASK", - [JOB_CONTROL__JS10__HEAD_NEXT] = "JOB_CONTROL__JS10__HEAD_NEXT", - [JOB_CONTROL__JS10__TAIL_NEXT] = "JOB_CONTROL__JS10__TAIL_NEXT", - [JOB_CONTROL__JS10__AFFINITY_NEXT] = "JOB_CONTROL__JS10__AFFINITY_NEXT", - [JOB_CONTROL__JS10__CONFIG_NEXT] = "JOB_CONTROL__JS10__CONFIG_NEXT", - [JOB_CONTROL__JS10__XAFFINITY_NEXT] = "JOB_CONTROL__JS10__XAFFINITY_NEXT", - [JOB_CONTROL__JS10__COMMAND_NEXT] = "JOB_CONTROL__JS10__COMMAND_NEXT", - [JOB_CONTROL__JS10__FLUSH_ID_NEXT] = "JOB_CONTROL__JS10__FLUSH_ID_NEXT", - [JOB_CONTROL__JS10__EVENT_MASK_NEXT] = "JOB_CONTROL__JS10__EVENT_MASK_NEXT", - [JOB_CONTROL__JS11__HEAD] = "JOB_CONTROL__JS11__HEAD", - [JOB_CONTROL__JS11__TAIL] = "JOB_CONTROL__JS11__TAIL", - [JOB_CONTROL__JS11__AFFINITY] = "JOB_CONTROL__JS11__AFFINITY", - [JOB_CONTROL__JS11__CONFIG] = "JOB_CONTROL__JS11__CONFIG", - [JOB_CONTROL__JS11__XAFFINITY] = "JOB_CONTROL__JS11__XAFFINITY", - [JOB_CONTROL__JS11__COMMAND] = "JOB_CONTROL__JS11__COMMAND", - [JOB_CONTROL__JS11__STATUS] = "JOB_CONTROL__JS11__STATUS", - [JOB_CONTROL__JS11__FLUSH_ID] = "JOB_CONTROL__JS11__FLUSH_ID", - [JOB_CONTROL__JS11__EVENT_0] = "JOB_CONTROL__JS11__EVENT_0", - [JOB_CONTROL__JS11__EVENT_1] = "JOB_CONTROL__JS11__EVENT_1", - [JOB_CONTROL__JS11__EVENT_MASK] = "JOB_CONTROL__JS11__EVENT_MASK", - [JOB_CONTROL__JS11__HEAD_NEXT] = "JOB_CONTROL__JS11__HEAD_NEXT", - [JOB_CONTROL__JS11__TAIL_NEXT] = "JOB_CONTROL__JS11__TAIL_NEXT", - [JOB_CONTROL__JS11__AFFINITY_NEXT] = "JOB_CONTROL__JS11__AFFINITY_NEXT", - [JOB_CONTROL__JS11__CONFIG_NEXT] = "JOB_CONTROL__JS11__CONFIG_NEXT", - [JOB_CONTROL__JS11__XAFFINITY_NEXT] = "JOB_CONTROL__JS11__XAFFINITY_NEXT", - [JOB_CONTROL__JS11__COMMAND_NEXT] = "JOB_CONTROL__JS11__COMMAND_NEXT", - [JOB_CONTROL__JS11__FLUSH_ID_NEXT] = "JOB_CONTROL__JS11__FLUSH_ID_NEXT", - [JOB_CONTROL__JS11__EVENT_MASK_NEXT] = "JOB_CONTROL__JS11__EVENT_MASK_NEXT", - [JOB_CONTROL__JS12__HEAD] = "JOB_CONTROL__JS12__HEAD", - [JOB_CONTROL__JS12__TAIL] = "JOB_CONTROL__JS12__TAIL", - [JOB_CONTROL__JS12__AFFINITY] = "JOB_CONTROL__JS12__AFFINITY", - [JOB_CONTROL__JS12__CONFIG] = "JOB_CONTROL__JS12__CONFIG", - [JOB_CONTROL__JS12__XAFFINITY] = "JOB_CONTROL__JS12__XAFFINITY", - [JOB_CONTROL__JS12__COMMAND] = "JOB_CONTROL__JS12__COMMAND", - [JOB_CONTROL__JS12__STATUS] = "JOB_CONTROL__JS12__STATUS", - [JOB_CONTROL__JS12__FLUSH_ID] = "JOB_CONTROL__JS12__FLUSH_ID", - [JOB_CONTROL__JS12__EVENT_0] = "JOB_CONTROL__JS12__EVENT_0", - [JOB_CONTROL__JS12__EVENT_1] = "JOB_CONTROL__JS12__EVENT_1", - [JOB_CONTROL__JS12__EVENT_MASK] = "JOB_CONTROL__JS12__EVENT_MASK", - [JOB_CONTROL__JS12__HEAD_NEXT] = "JOB_CONTROL__JS12__HEAD_NEXT", - [JOB_CONTROL__JS12__TAIL_NEXT] = "JOB_CONTROL__JS12__TAIL_NEXT", - [JOB_CONTROL__JS12__AFFINITY_NEXT] = "JOB_CONTROL__JS12__AFFINITY_NEXT", - [JOB_CONTROL__JS12__CONFIG_NEXT] = "JOB_CONTROL__JS12__CONFIG_NEXT", - [JOB_CONTROL__JS12__XAFFINITY_NEXT] = "JOB_CONTROL__JS12__XAFFINITY_NEXT", - [JOB_CONTROL__JS12__COMMAND_NEXT] = "JOB_CONTROL__JS12__COMMAND_NEXT", - [JOB_CONTROL__JS12__FLUSH_ID_NEXT] = "JOB_CONTROL__JS12__FLUSH_ID_NEXT", - [JOB_CONTROL__JS12__EVENT_MASK_NEXT] = "JOB_CONTROL__JS12__EVENT_MASK_NEXT", - [JOB_CONTROL__JS13__HEAD] = "JOB_CONTROL__JS13__HEAD", - [JOB_CONTROL__JS13__TAIL] = "JOB_CONTROL__JS13__TAIL", - [JOB_CONTROL__JS13__AFFINITY] = "JOB_CONTROL__JS13__AFFINITY", - [JOB_CONTROL__JS13__CONFIG] = "JOB_CONTROL__JS13__CONFIG", - [JOB_CONTROL__JS13__XAFFINITY] = "JOB_CONTROL__JS13__XAFFINITY", - [JOB_CONTROL__JS13__COMMAND] = "JOB_CONTROL__JS13__COMMAND", - [JOB_CONTROL__JS13__STATUS] = "JOB_CONTROL__JS13__STATUS", - [JOB_CONTROL__JS13__FLUSH_ID] = "JOB_CONTROL__JS13__FLUSH_ID", - [JOB_CONTROL__JS13__EVENT_0] = "JOB_CONTROL__JS13__EVENT_0", - [JOB_CONTROL__JS13__EVENT_1] = "JOB_CONTROL__JS13__EVENT_1", - [JOB_CONTROL__JS13__EVENT_MASK] = "JOB_CONTROL__JS13__EVENT_MASK", - [JOB_CONTROL__JS13__HEAD_NEXT] = "JOB_CONTROL__JS13__HEAD_NEXT", - [JOB_CONTROL__JS13__TAIL_NEXT] = "JOB_CONTROL__JS13__TAIL_NEXT", - [JOB_CONTROL__JS13__AFFINITY_NEXT] = "JOB_CONTROL__JS13__AFFINITY_NEXT", - [JOB_CONTROL__JS13__CONFIG_NEXT] = "JOB_CONTROL__JS13__CONFIG_NEXT", - [JOB_CONTROL__JS13__XAFFINITY_NEXT] = "JOB_CONTROL__JS13__XAFFINITY_NEXT", - [JOB_CONTROL__JS13__COMMAND_NEXT] = "JOB_CONTROL__JS13__COMMAND_NEXT", - [JOB_CONTROL__JS13__FLUSH_ID_NEXT] = "JOB_CONTROL__JS13__FLUSH_ID_NEXT", - [JOB_CONTROL__JS13__EVENT_MASK_NEXT] = "JOB_CONTROL__JS13__EVENT_MASK_NEXT", - [JOB_CONTROL__JS14__HEAD] = "JOB_CONTROL__JS14__HEAD", - [JOB_CONTROL__JS14__TAIL] = "JOB_CONTROL__JS14__TAIL", - [JOB_CONTROL__JS14__AFFINITY] = "JOB_CONTROL__JS14__AFFINITY", - [JOB_CONTROL__JS14__CONFIG] = "JOB_CONTROL__JS14__CONFIG", - [JOB_CONTROL__JS14__XAFFINITY] = "JOB_CONTROL__JS14__XAFFINITY", - [JOB_CONTROL__JS14__COMMAND] = "JOB_CONTROL__JS14__COMMAND", - [JOB_CONTROL__JS14__STATUS] = "JOB_CONTROL__JS14__STATUS", - [JOB_CONTROL__JS14__FLUSH_ID] = "JOB_CONTROL__JS14__FLUSH_ID", - [JOB_CONTROL__JS14__EVENT_0] = "JOB_CONTROL__JS14__EVENT_0", - [JOB_CONTROL__JS14__EVENT_1] = "JOB_CONTROL__JS14__EVENT_1", - [JOB_CONTROL__JS14__EVENT_MASK] = "JOB_CONTROL__JS14__EVENT_MASK", - [JOB_CONTROL__JS14__HEAD_NEXT] = "JOB_CONTROL__JS14__HEAD_NEXT", - [JOB_CONTROL__JS14__TAIL_NEXT] = "JOB_CONTROL__JS14__TAIL_NEXT", - [JOB_CONTROL__JS14__AFFINITY_NEXT] = "JOB_CONTROL__JS14__AFFINITY_NEXT", - [JOB_CONTROL__JS14__CONFIG_NEXT] = "JOB_CONTROL__JS14__CONFIG_NEXT", - [JOB_CONTROL__JS14__XAFFINITY_NEXT] = "JOB_CONTROL__JS14__XAFFINITY_NEXT", - [JOB_CONTROL__JS14__COMMAND_NEXT] = "JOB_CONTROL__JS14__COMMAND_NEXT", - [JOB_CONTROL__JS14__FLUSH_ID_NEXT] = "JOB_CONTROL__JS14__FLUSH_ID_NEXT", - [JOB_CONTROL__JS14__EVENT_MASK_NEXT] = "JOB_CONTROL__JS14__EVENT_MASK_NEXT", - [JOB_CONTROL__JS15__HEAD] = "JOB_CONTROL__JS15__HEAD", - [JOB_CONTROL__JS15__TAIL] = "JOB_CONTROL__JS15__TAIL", - [JOB_CONTROL__JS15__AFFINITY] = "JOB_CONTROL__JS15__AFFINITY", - [JOB_CONTROL__JS15__CONFIG] = "JOB_CONTROL__JS15__CONFIG", - [JOB_CONTROL__JS15__XAFFINITY] = "JOB_CONTROL__JS15__XAFFINITY", - [JOB_CONTROL__JS15__COMMAND] = "JOB_CONTROL__JS15__COMMAND", - [JOB_CONTROL__JS15__STATUS] = "JOB_CONTROL__JS15__STATUS", - [JOB_CONTROL__JS15__FLUSH_ID] = "JOB_CONTROL__JS15__FLUSH_ID", - [JOB_CONTROL__JS15__EVENT_0] = "JOB_CONTROL__JS15__EVENT_0", - [JOB_CONTROL__JS15__EVENT_1] = "JOB_CONTROL__JS15__EVENT_1", - [JOB_CONTROL__JS15__EVENT_MASK] = "JOB_CONTROL__JS15__EVENT_MASK", - [JOB_CONTROL__JS15__HEAD_NEXT] = "JOB_CONTROL__JS15__HEAD_NEXT", - [JOB_CONTROL__JS15__TAIL_NEXT] = "JOB_CONTROL__JS15__TAIL_NEXT", - [JOB_CONTROL__JS15__AFFINITY_NEXT] = "JOB_CONTROL__JS15__AFFINITY_NEXT", - [JOB_CONTROL__JS15__CONFIG_NEXT] = "JOB_CONTROL__JS15__CONFIG_NEXT", - [JOB_CONTROL__JS15__XAFFINITY_NEXT] = "JOB_CONTROL__JS15__XAFFINITY_NEXT", - [JOB_CONTROL__JS15__COMMAND_NEXT] = "JOB_CONTROL__JS15__COMMAND_NEXT", - [JOB_CONTROL__JS15__FLUSH_ID_NEXT] = "JOB_CONTROL__JS15__FLUSH_ID_NEXT", - [JOB_CONTROL__JS15__EVENT_MASK_NEXT] = "JOB_CONTROL__JS15__EVENT_MASK_NEXT", - [MMU_STAGE1__ST1MMU__IRQ_RAWSTAT] = "MMU_STAGE1__ST1MMU__IRQ_RAWSTAT", - [MMU_STAGE1__ST1MMU__IRQ_CLEAR] = "MMU_STAGE1__ST1MMU__IRQ_CLEAR", - [MMU_STAGE1__ST1MMU__IRQ_MASK] = "MMU_STAGE1__ST1MMU__IRQ_MASK", - [MMU_STAGE1__ST1MMU__IRQ_STATUS] = "MMU_STAGE1__ST1MMU__IRQ_STATUS", - [MMU_STAGE1__ST1MMU__AS0__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS0__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS0__MEMATTR] = "MMU_STAGE1__ST1MMU__AS0__MEMATTR", - [MMU_STAGE1__ST1MMU__AS0__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS0__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS0__COMMAND] = "MMU_STAGE1__ST1MMU__AS0__COMMAND", - [MMU_STAGE1__ST1MMU__AS0__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS0__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS0__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS0__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS0__STATUS] = "MMU_STAGE1__ST1MMU__AS0__STATUS", - [MMU_STAGE1__ST1MMU__AS0__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS0__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS0__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS0__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS1__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS1__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS1__MEMATTR] = "MMU_STAGE1__ST1MMU__AS1__MEMATTR", - [MMU_STAGE1__ST1MMU__AS1__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS1__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS1__COMMAND] = "MMU_STAGE1__ST1MMU__AS1__COMMAND", - [MMU_STAGE1__ST1MMU__AS1__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS1__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS1__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS1__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS1__STATUS] = "MMU_STAGE1__ST1MMU__AS1__STATUS", - [MMU_STAGE1__ST1MMU__AS1__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS1__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS1__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS1__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS2__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS2__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS2__MEMATTR] = "MMU_STAGE1__ST1MMU__AS2__MEMATTR", - [MMU_STAGE1__ST1MMU__AS2__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS2__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS2__COMMAND] = "MMU_STAGE1__ST1MMU__AS2__COMMAND", - [MMU_STAGE1__ST1MMU__AS2__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS2__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS2__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS2__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS2__STATUS] = "MMU_STAGE1__ST1MMU__AS2__STATUS", - [MMU_STAGE1__ST1MMU__AS2__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS2__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS2__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS2__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS3__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS3__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS3__MEMATTR] = "MMU_STAGE1__ST1MMU__AS3__MEMATTR", - [MMU_STAGE1__ST1MMU__AS3__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS3__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS3__COMMAND] = "MMU_STAGE1__ST1MMU__AS3__COMMAND", - [MMU_STAGE1__ST1MMU__AS3__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS3__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS3__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS3__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS3__STATUS] = "MMU_STAGE1__ST1MMU__AS3__STATUS", - [MMU_STAGE1__ST1MMU__AS3__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS3__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS3__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS3__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS4__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS4__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS4__MEMATTR] = "MMU_STAGE1__ST1MMU__AS4__MEMATTR", - [MMU_STAGE1__ST1MMU__AS4__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS4__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS4__COMMAND] = "MMU_STAGE1__ST1MMU__AS4__COMMAND", - [MMU_STAGE1__ST1MMU__AS4__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS4__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS4__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS4__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS4__STATUS] = "MMU_STAGE1__ST1MMU__AS4__STATUS", - [MMU_STAGE1__ST1MMU__AS4__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS4__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS4__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS4__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS5__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS5__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS5__MEMATTR] = "MMU_STAGE1__ST1MMU__AS5__MEMATTR", - [MMU_STAGE1__ST1MMU__AS5__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS5__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS5__COMMAND] = "MMU_STAGE1__ST1MMU__AS5__COMMAND", - [MMU_STAGE1__ST1MMU__AS5__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS5__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS5__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS5__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS5__STATUS] = "MMU_STAGE1__ST1MMU__AS5__STATUS", - [MMU_STAGE1__ST1MMU__AS5__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS5__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS5__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS5__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS6__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS6__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS6__MEMATTR] = "MMU_STAGE1__ST1MMU__AS6__MEMATTR", - [MMU_STAGE1__ST1MMU__AS6__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS6__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS6__COMMAND] = "MMU_STAGE1__ST1MMU__AS6__COMMAND", - [MMU_STAGE1__ST1MMU__AS6__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS6__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS6__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS6__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS6__STATUS] = "MMU_STAGE1__ST1MMU__AS6__STATUS", - [MMU_STAGE1__ST1MMU__AS6__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS6__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS6__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS6__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS7__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS7__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS7__MEMATTR] = "MMU_STAGE1__ST1MMU__AS7__MEMATTR", - [MMU_STAGE1__ST1MMU__AS7__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS7__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS7__COMMAND] = "MMU_STAGE1__ST1MMU__AS7__COMMAND", - [MMU_STAGE1__ST1MMU__AS7__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS7__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS7__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS7__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS7__STATUS] = "MMU_STAGE1__ST1MMU__AS7__STATUS", - [MMU_STAGE1__ST1MMU__AS7__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS7__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS7__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS7__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS8__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS8__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS8__MEMATTR] = "MMU_STAGE1__ST1MMU__AS8__MEMATTR", - [MMU_STAGE1__ST1MMU__AS8__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS8__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS8__COMMAND] = "MMU_STAGE1__ST1MMU__AS8__COMMAND", - [MMU_STAGE1__ST1MMU__AS8__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS8__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS8__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS8__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS8__STATUS] = "MMU_STAGE1__ST1MMU__AS8__STATUS", - [MMU_STAGE1__ST1MMU__AS8__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS8__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS8__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS8__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS9__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS9__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS9__MEMATTR] = "MMU_STAGE1__ST1MMU__AS9__MEMATTR", - [MMU_STAGE1__ST1MMU__AS9__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS9__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS9__COMMAND] = "MMU_STAGE1__ST1MMU__AS9__COMMAND", - [MMU_STAGE1__ST1MMU__AS9__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS9__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS9__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS9__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS9__STATUS] = "MMU_STAGE1__ST1MMU__AS9__STATUS", - [MMU_STAGE1__ST1MMU__AS9__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS9__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS9__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS9__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS10__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS10__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS10__MEMATTR] = "MMU_STAGE1__ST1MMU__AS10__MEMATTR", - [MMU_STAGE1__ST1MMU__AS10__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS10__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS10__COMMAND] = "MMU_STAGE1__ST1MMU__AS10__COMMAND", - [MMU_STAGE1__ST1MMU__AS10__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS10__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS10__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS10__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS10__STATUS] = "MMU_STAGE1__ST1MMU__AS10__STATUS", - [MMU_STAGE1__ST1MMU__AS10__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS10__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS10__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS10__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS11__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS11__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS11__MEMATTR] = "MMU_STAGE1__ST1MMU__AS11__MEMATTR", - [MMU_STAGE1__ST1MMU__AS11__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS11__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS11__COMMAND] = "MMU_STAGE1__ST1MMU__AS11__COMMAND", - [MMU_STAGE1__ST1MMU__AS11__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS11__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS11__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS11__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS11__STATUS] = "MMU_STAGE1__ST1MMU__AS11__STATUS", - [MMU_STAGE1__ST1MMU__AS11__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS11__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS11__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS11__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS12__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS12__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS12__MEMATTR] = "MMU_STAGE1__ST1MMU__AS12__MEMATTR", - [MMU_STAGE1__ST1MMU__AS12__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS12__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS12__COMMAND] = "MMU_STAGE1__ST1MMU__AS12__COMMAND", - [MMU_STAGE1__ST1MMU__AS12__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS12__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS12__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS12__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS12__STATUS] = "MMU_STAGE1__ST1MMU__AS12__STATUS", - [MMU_STAGE1__ST1MMU__AS12__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS12__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS12__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS12__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS13__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS13__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS13__MEMATTR] = "MMU_STAGE1__ST1MMU__AS13__MEMATTR", - [MMU_STAGE1__ST1MMU__AS13__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS13__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS13__COMMAND] = "MMU_STAGE1__ST1MMU__AS13__COMMAND", - [MMU_STAGE1__ST1MMU__AS13__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS13__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS13__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS13__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS13__STATUS] = "MMU_STAGE1__ST1MMU__AS13__STATUS", - [MMU_STAGE1__ST1MMU__AS13__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS13__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS13__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS13__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS14__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS14__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS14__MEMATTR] = "MMU_STAGE1__ST1MMU__AS14__MEMATTR", - [MMU_STAGE1__ST1MMU__AS14__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS14__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS14__COMMAND] = "MMU_STAGE1__ST1MMU__AS14__COMMAND", - [MMU_STAGE1__ST1MMU__AS14__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS14__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS14__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS14__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS14__STATUS] = "MMU_STAGE1__ST1MMU__AS14__STATUS", - [MMU_STAGE1__ST1MMU__AS14__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS14__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS14__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS14__FAULTEXTRA", - [MMU_STAGE1__ST1MMU__AS15__TRANSTAB] = "MMU_STAGE1__ST1MMU__AS15__TRANSTAB", - [MMU_STAGE1__ST1MMU__AS15__MEMATTR] = "MMU_STAGE1__ST1MMU__AS15__MEMATTR", - [MMU_STAGE1__ST1MMU__AS15__LOCKADDR] = "MMU_STAGE1__ST1MMU__AS15__LOCKADDR", - [MMU_STAGE1__ST1MMU__AS15__COMMAND] = "MMU_STAGE1__ST1MMU__AS15__COMMAND", - [MMU_STAGE1__ST1MMU__AS15__FAULTSTATUS] = "MMU_STAGE1__ST1MMU__AS15__FAULTSTATUS", - [MMU_STAGE1__ST1MMU__AS15__FAULTADDRESS] = "MMU_STAGE1__ST1MMU__AS15__FAULTADDRESS", - [MMU_STAGE1__ST1MMU__AS15__STATUS] = "MMU_STAGE1__ST1MMU__AS15__STATUS", - [MMU_STAGE1__ST1MMU__AS15__TRANSCFG] = "MMU_STAGE1__ST1MMU__AS15__TRANSCFG", - [MMU_STAGE1__ST1MMU__AS15__FAULTEXTRA] = "MMU_STAGE1__ST1MMU__AS15__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__IRQ_RAWSTAT] = "MMU_STAGE2__ST2MMU__IRQ_RAWSTAT", - [MMU_STAGE2__ST2MMU__IRQ_CLEAR] = "MMU_STAGE2__ST2MMU__IRQ_CLEAR", - [MMU_STAGE2__ST2MMU__IRQ_MASK] = "MMU_STAGE2__ST2MMU__IRQ_MASK", - [MMU_STAGE2__ST2MMU__IRQ_STATUS] = "MMU_STAGE2__ST2MMU__IRQ_STATUS", - [MMU_STAGE2__ST2MMU__AS0__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS0__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS0__MEMATTR] = "MMU_STAGE2__ST2MMU__AS0__MEMATTR", - [MMU_STAGE2__ST2MMU__AS0__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS0__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS0__COMMAND] = "MMU_STAGE2__ST2MMU__AS0__COMMAND", - [MMU_STAGE2__ST2MMU__AS0__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS0__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS0__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS0__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS0__STATUS] = "MMU_STAGE2__ST2MMU__AS0__STATUS", - [MMU_STAGE2__ST2MMU__AS0__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS0__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS0__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS0__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS1__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS1__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS1__MEMATTR] = "MMU_STAGE2__ST2MMU__AS1__MEMATTR", - [MMU_STAGE2__ST2MMU__AS1__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS1__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS1__COMMAND] = "MMU_STAGE2__ST2MMU__AS1__COMMAND", - [MMU_STAGE2__ST2MMU__AS1__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS1__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS1__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS1__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS1__STATUS] = "MMU_STAGE2__ST2MMU__AS1__STATUS", - [MMU_STAGE2__ST2MMU__AS1__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS1__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS1__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS1__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS2__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS2__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS2__MEMATTR] = "MMU_STAGE2__ST2MMU__AS2__MEMATTR", - [MMU_STAGE2__ST2MMU__AS2__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS2__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS2__COMMAND] = "MMU_STAGE2__ST2MMU__AS2__COMMAND", - [MMU_STAGE2__ST2MMU__AS2__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS2__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS2__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS2__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS2__STATUS] = "MMU_STAGE2__ST2MMU__AS2__STATUS", - [MMU_STAGE2__ST2MMU__AS2__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS2__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS2__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS2__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS3__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS3__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS3__MEMATTR] = "MMU_STAGE2__ST2MMU__AS3__MEMATTR", - [MMU_STAGE2__ST2MMU__AS3__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS3__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS3__COMMAND] = "MMU_STAGE2__ST2MMU__AS3__COMMAND", - [MMU_STAGE2__ST2MMU__AS3__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS3__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS3__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS3__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS3__STATUS] = "MMU_STAGE2__ST2MMU__AS3__STATUS", - [MMU_STAGE2__ST2MMU__AS3__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS3__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS3__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS3__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS4__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS4__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS4__MEMATTR] = "MMU_STAGE2__ST2MMU__AS4__MEMATTR", - [MMU_STAGE2__ST2MMU__AS4__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS4__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS4__COMMAND] = "MMU_STAGE2__ST2MMU__AS4__COMMAND", - [MMU_STAGE2__ST2MMU__AS4__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS4__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS4__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS4__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS4__STATUS] = "MMU_STAGE2__ST2MMU__AS4__STATUS", - [MMU_STAGE2__ST2MMU__AS4__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS4__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS4__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS4__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS5__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS5__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS5__MEMATTR] = "MMU_STAGE2__ST2MMU__AS5__MEMATTR", - [MMU_STAGE2__ST2MMU__AS5__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS5__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS5__COMMAND] = "MMU_STAGE2__ST2MMU__AS5__COMMAND", - [MMU_STAGE2__ST2MMU__AS5__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS5__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS5__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS5__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS5__STATUS] = "MMU_STAGE2__ST2MMU__AS5__STATUS", - [MMU_STAGE2__ST2MMU__AS5__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS5__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS5__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS5__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS6__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS6__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS6__MEMATTR] = "MMU_STAGE2__ST2MMU__AS6__MEMATTR", - [MMU_STAGE2__ST2MMU__AS6__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS6__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS6__COMMAND] = "MMU_STAGE2__ST2MMU__AS6__COMMAND", - [MMU_STAGE2__ST2MMU__AS6__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS6__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS6__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS6__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS6__STATUS] = "MMU_STAGE2__ST2MMU__AS6__STATUS", - [MMU_STAGE2__ST2MMU__AS6__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS6__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS6__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS6__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS7__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS7__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS7__MEMATTR] = "MMU_STAGE2__ST2MMU__AS7__MEMATTR", - [MMU_STAGE2__ST2MMU__AS7__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS7__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS7__COMMAND] = "MMU_STAGE2__ST2MMU__AS7__COMMAND", - [MMU_STAGE2__ST2MMU__AS7__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS7__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS7__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS7__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS7__STATUS] = "MMU_STAGE2__ST2MMU__AS7__STATUS", - [MMU_STAGE2__ST2MMU__AS7__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS7__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS7__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS7__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS8__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS8__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS8__MEMATTR] = "MMU_STAGE2__ST2MMU__AS8__MEMATTR", - [MMU_STAGE2__ST2MMU__AS8__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS8__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS8__COMMAND] = "MMU_STAGE2__ST2MMU__AS8__COMMAND", - [MMU_STAGE2__ST2MMU__AS8__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS8__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS8__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS8__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS8__STATUS] = "MMU_STAGE2__ST2MMU__AS8__STATUS", - [MMU_STAGE2__ST2MMU__AS8__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS8__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS8__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS8__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS9__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS9__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS9__MEMATTR] = "MMU_STAGE2__ST2MMU__AS9__MEMATTR", - [MMU_STAGE2__ST2MMU__AS9__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS9__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS9__COMMAND] = "MMU_STAGE2__ST2MMU__AS9__COMMAND", - [MMU_STAGE2__ST2MMU__AS9__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS9__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS9__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS9__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS9__STATUS] = "MMU_STAGE2__ST2MMU__AS9__STATUS", - [MMU_STAGE2__ST2MMU__AS9__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS9__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS9__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS9__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS10__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS10__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS10__MEMATTR] = "MMU_STAGE2__ST2MMU__AS10__MEMATTR", - [MMU_STAGE2__ST2MMU__AS10__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS10__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS10__COMMAND] = "MMU_STAGE2__ST2MMU__AS10__COMMAND", - [MMU_STAGE2__ST2MMU__AS10__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS10__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS10__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS10__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS10__STATUS] = "MMU_STAGE2__ST2MMU__AS10__STATUS", - [MMU_STAGE2__ST2MMU__AS10__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS10__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS10__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS10__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS11__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS11__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS11__MEMATTR] = "MMU_STAGE2__ST2MMU__AS11__MEMATTR", - [MMU_STAGE2__ST2MMU__AS11__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS11__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS11__COMMAND] = "MMU_STAGE2__ST2MMU__AS11__COMMAND", - [MMU_STAGE2__ST2MMU__AS11__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS11__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS11__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS11__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS11__STATUS] = "MMU_STAGE2__ST2MMU__AS11__STATUS", - [MMU_STAGE2__ST2MMU__AS11__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS11__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS11__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS11__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS12__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS12__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS12__MEMATTR] = "MMU_STAGE2__ST2MMU__AS12__MEMATTR", - [MMU_STAGE2__ST2MMU__AS12__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS12__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS12__COMMAND] = "MMU_STAGE2__ST2MMU__AS12__COMMAND", - [MMU_STAGE2__ST2MMU__AS12__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS12__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS12__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS12__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS12__STATUS] = "MMU_STAGE2__ST2MMU__AS12__STATUS", - [MMU_STAGE2__ST2MMU__AS12__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS12__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS12__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS12__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS13__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS13__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS13__MEMATTR] = "MMU_STAGE2__ST2MMU__AS13__MEMATTR", - [MMU_STAGE2__ST2MMU__AS13__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS13__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS13__COMMAND] = "MMU_STAGE2__ST2MMU__AS13__COMMAND", - [MMU_STAGE2__ST2MMU__AS13__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS13__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS13__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS13__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS13__STATUS] = "MMU_STAGE2__ST2MMU__AS13__STATUS", - [MMU_STAGE2__ST2MMU__AS13__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS13__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS13__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS13__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS14__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS14__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS14__MEMATTR] = "MMU_STAGE2__ST2MMU__AS14__MEMATTR", - [MMU_STAGE2__ST2MMU__AS14__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS14__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS14__COMMAND] = "MMU_STAGE2__ST2MMU__AS14__COMMAND", - [MMU_STAGE2__ST2MMU__AS14__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS14__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS14__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS14__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS14__STATUS] = "MMU_STAGE2__ST2MMU__AS14__STATUS", - [MMU_STAGE2__ST2MMU__AS14__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS14__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS14__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS14__FAULTEXTRA", - [MMU_STAGE2__ST2MMU__AS15__TRANSTAB] = "MMU_STAGE2__ST2MMU__AS15__TRANSTAB", - [MMU_STAGE2__ST2MMU__AS15__MEMATTR] = "MMU_STAGE2__ST2MMU__AS15__MEMATTR", - [MMU_STAGE2__ST2MMU__AS15__LOCKADDR] = "MMU_STAGE2__ST2MMU__AS15__LOCKADDR", - [MMU_STAGE2__ST2MMU__AS15__COMMAND] = "MMU_STAGE2__ST2MMU__AS15__COMMAND", - [MMU_STAGE2__ST2MMU__AS15__FAULTSTATUS] = "MMU_STAGE2__ST2MMU__AS15__FAULTSTATUS", - [MMU_STAGE2__ST2MMU__AS15__FAULTADDRESS] = "MMU_STAGE2__ST2MMU__AS15__FAULTADDRESS", - [MMU_STAGE2__ST2MMU__AS15__STATUS] = "MMU_STAGE2__ST2MMU__AS15__STATUS", - [MMU_STAGE2__ST2MMU__AS15__TRANSCFG] = "MMU_STAGE2__ST2MMU__AS15__TRANSCFG", - [MMU_STAGE2__ST2MMU__AS15__FAULTEXTRA] = "MMU_STAGE2__ST2MMU__AS15__FAULTEXTRA", - [GPU_CONTROL__REVIDR] = "GPU_CONTROL__REVIDR", - [GPU_CONTROL__STACK_PRESENT] = "GPU_CONTROL__STACK_PRESENT", - [GPU_CONTROL__STACK_PWROFF] = "GPU_CONTROL__STACK_PWROFF", - [GPU_CONTROL__STACK_PWRON] = "GPU_CONTROL__STACK_PWRON", - [GPU_CONTROL__STACK_PWRTRANS] = "GPU_CONTROL__STACK_PWRTRANS", - [GPU_CONTROL__STACK_READY] = "GPU_CONTROL__STACK_READY", - [GPU_CONTROL__TEXTURE_FEATURES_3] = "GPU_CONTROL__TEXTURE_FEATURES_3", - [GPU_CONTROL__CORE_FEATURES] = "GPU_CONTROL__CORE_FEATURES", - [GPU_CONTROL__THREAD_TLS_ALLOC] = "GPU_CONTROL__THREAD_TLS_ALLOC", - [GPU_CONTROL__L2_CONFIG] = "GPU_CONTROL__L2_CONFIG", - [PTM_AW_IRQ_CLEAR] = "PTM_AW_IRQ_CLEAR", - [PTM_AW_IRQ_INJECTION] = "PTM_AW_IRQ_INJECTION", - [PTM_AW_IRQ_MASK] = "PTM_AW_IRQ_MASK", - [PTM_AW_IRQ_RAWSTAT] = "PTM_AW_IRQ_RAWSTAT", - [PTM_AW_IRQ_STATUS] = "PTM_AW_IRQ_STATUS", - [PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0] = "PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0", - [PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1] = "PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1", - [PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0] = "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0", - [PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1] = "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1", - [PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS] = - "PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS", - [PTM_ID] = "PTM_ID", -}; - -const char *kbase_reg_get_enum_string(u32 reg_enum) -{ - if (reg_enum >= ARRAY_SIZE(enum_strings)) - return "INVALID_REG"; - return enum_strings[reg_enum]; -} -#endif /* CONFIG_MALI_VALHALL_DEBUG */ diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_enums.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_enums.h deleted file mode 100644 index 59d8745eaf4a..000000000000 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_enums.h +++ /dev/null @@ -1,777 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * This header is autogenerated. Avoid modifying this file unless absolutely - * necessary. - */ - -#ifndef _MALI_KBASE_REGMAP_JM_ENUMS_H_ -#define _MALI_KBASE_REGMAP_JM_ENUMS_H_ - -#if MALI_USE_CSF -#error "Cannot be compiled with CSF" -#endif - -enum kbase_regmap_enum_v6_0 { - GPU_CONTROL__GPU_ID = 0, /* (RO) 32-bit 0x0 */ - GPU_CONTROL__L2_FEATURES, /* (RO) 32-bit 0x4 */ - GPU_CONTROL__TILER_FEATURES, /* (RO) 32-bit 0xC */ - GPU_CONTROL__MEM_FEATURES, /* (RO) 32-bit 0x10 */ - GPU_CONTROL__MMU_FEATURES, /* (RO) 32-bit 0x14 */ - GPU_CONTROL__AS_PRESENT, /* (RO) 32-bit 0x18 */ - GPU_CONTROL__JS_PRESENT, /* (RO) 32-bit 0x1C */ - GPU_CONTROL__GPU_IRQ_RAWSTAT, /* (RW) 32-bit 0x20 */ - GPU_CONTROL__GPU_IRQ_CLEAR, /* (WO) 32-bit 0x24 */ - GPU_CONTROL__GPU_IRQ_MASK, /* (RW) 32-bit 0x28 */ - GPU_CONTROL__GPU_IRQ_STATUS, /* (RO) 32-bit 0x2C */ - GPU_CONTROL__GPU_COMMAND, /* (WO) 32-bit 0x30 */ - GPU_CONTROL__GPU_STATUS, /* (RO) 32-bit 0x34 */ - GPU_CONTROL__LATEST_FLUSH, /* (RO) 32-bit 0x38 */ - GPU_CONTROL__GPU_FAULTSTATUS, /* (RO) 32-bit 0x3C */ - GPU_CONTROL__GPU_FAULTADDRESS, /* (RO) 64-bit 0x40 */ - GPU_CONTROL__AFBC_FEATURES, /* (RO) 32-bit 0x4C */ - GPU_CONTROL__PWR_KEY, /* (WO) 32-bit 0x50 */ - GPU_CONTROL__PWR_OVERRIDE0, /* (RW) 32-bit 0x54 */ - GPU_CONTROL__PWR_OVERRIDE1, /* (RW) 32-bit 0x58 */ - GPU_CONTROL__PRFCNT_BASE, /* (RW) 64-bit 0x60 */ - GPU_CONTROL__PRFCNT_CONFIG, /* (RW) 32-bit 0x68 */ - GPU_CONTROL__PRFCNT_JM_EN, /* (RW) 32-bit 0x6C */ - GPU_CONTROL__PRFCNT_SHADER_EN, /* (RW) 32-bit 0x70 */ - GPU_CONTROL__PRFCNT_TILER_EN, /* (RW) 32-bit 0x74 */ - GPU_CONTROL__PRFCNT_MMU_L2_EN, /* (RW) 32-bit 0x7C */ - GPU_CONTROL__CYCLE_COUNT, /* (RO) 64-bit 0x90 */ - GPU_CONTROL__TIMESTAMP, /* (RO) 64-bit 0x98 */ - GPU_CONTROL__THREAD_MAX_THREADS, /* (RO) 32-bit 0xA0 */ - GPU_CONTROL__THREAD_MAX_WORKGROUP_SIZE, /* (RO) 32-bit 0xA4 */ - GPU_CONTROL__THREAD_MAX_BARRIER_SIZE, /* (RO) 32-bit 0xA8 */ - GPU_CONTROL__THREAD_FEATURES, /* (RO) 32-bit 0xAC */ - GPU_CONTROL__TEXTURE_FEATURES_0, /* (RO) 32-bit 0xB0 */ - GPU_CONTROL__TEXTURE_FEATURES_1, /* (RO) 32-bit 0xB4 */ - GPU_CONTROL__TEXTURE_FEATURES_2, /* (RO) 32-bit 0xB8 */ - GPU_CONTROL__JS0_FEATURES, /* (RO) 32-bit 0xC0 */ - GPU_CONTROL__JS1_FEATURES, /* (RO) 32-bit 0xC4 */ - GPU_CONTROL__JS2_FEATURES, /* (RO) 32-bit 0xC8 */ - GPU_CONTROL__JS3_FEATURES, /* (RO) 32-bit 0xCC */ - GPU_CONTROL__JS4_FEATURES, /* (RO) 32-bit 0xD0 */ - GPU_CONTROL__JS5_FEATURES, /* (RO) 32-bit 0xD4 */ - GPU_CONTROL__JS6_FEATURES, /* (RO) 32-bit 0xD8 */ - GPU_CONTROL__JS7_FEATURES, /* (RO) 32-bit 0xDC */ - GPU_CONTROL__JS8_FEATURES, /* (RO) 32-bit 0xE0 */ - GPU_CONTROL__JS9_FEATURES, /* (RO) 32-bit 0xE4 */ - GPU_CONTROL__JS10_FEATURES, /* (RO) 32-bit 0xE8 */ - GPU_CONTROL__JS11_FEATURES, /* (RO) 32-bit 0xEC */ - GPU_CONTROL__JS12_FEATURES, /* (RO) 32-bit 0xF0 */ - GPU_CONTROL__JS13_FEATURES, /* (RO) 32-bit 0xF4 */ - GPU_CONTROL__JS14_FEATURES, /* (RO) 32-bit 0xF8 */ - GPU_CONTROL__JS15_FEATURES, /* (RO) 32-bit 0xFC */ - GPU_CONTROL__SHADER_PRESENT, /* (RO) 64-bit 0x100 */ - GPU_CONTROL__TILER_PRESENT, /* (RO) 64-bit 0x110 */ - GPU_CONTROL__L2_PRESENT, /* (RO) 64-bit 0x120 */ - GPU_CONTROL__SHADER_READY, /* (RO) 64-bit 0x140 */ - GPU_CONTROL__TILER_READY, /* (RO) 64-bit 0x150 */ - GPU_CONTROL__L2_READY, /* (RO) 64-bit 0x160 */ - GPU_CONTROL__SHADER_PWRON, /* (WO) 64-bit 0x180 */ - GPU_CONTROL__TILER_PWRON, /* (WO) 64-bit 0x190 */ - GPU_CONTROL__L2_PWRON, /* (WO) 64-bit 0x1A0 */ - GPU_CONTROL__SHADER_PWROFF, /* (WO) 64-bit 0x1C0 */ - GPU_CONTROL__TILER_PWROFF, /* (WO) 64-bit 0x1D0 */ - GPU_CONTROL__L2_PWROFF, /* (WO) 64-bit 0x1E0 */ - GPU_CONTROL__SHADER_PWRTRANS, /* (RO) 64-bit 0x200 */ - GPU_CONTROL__TILER_PWRTRANS, /* (RO) 64-bit 0x210 */ - GPU_CONTROL__L2_PWRTRANS, /* (RO) 64-bit 0x220 */ - GPU_CONTROL__SHADER_PWRACTIVE, /* (RO) 64-bit 0x240 */ - GPU_CONTROL__TILER_PWRACTIVE, /* (RO) 64-bit 0x250 */ - GPU_CONTROL__L2_PWRACTIVE, /* (RO) 64-bit 0x260 */ - GPU_CONTROL__COHERENCY_FEATURES, /* (RO) 32-bit 0x300 */ - GPU_CONTROL__COHERENCY_ENABLE, /* (RW) 32-bit 0x304 */ - GPU_CONTROL__GPU_USER_INn, /* (RO) 32-bit 0x400 */ - GPU_CONTROL__GPU_USER_OUTn, /* (RW) 32-bit 0x500 */ - GPU_CONTROL__JM_CONFIG, /* (RW) 32-bit 0xF00 */ - GPU_CONTROL__SHADER_CONFIG, /* (RW) 32-bit 0xF04 */ - GPU_CONTROL__TILER_CONFIG, /* (RW) 32-bit 0xF08 */ - GPU_CONTROL__L2_MMU_CONFIG, /* (RW) 32-bit 0xF0C */ - JOB_CONTROL__JOB_IRQ_RAWSTAT, /* (RW) 32-bit 0x1000 */ - JOB_CONTROL__JOB_IRQ_CLEAR, /* (WO) 32-bit 0x1004 */ - JOB_CONTROL__JOB_IRQ_MASK, /* (RW) 32-bit 0x1008 */ - JOB_CONTROL__JOB_IRQ_STATUS, /* (RO) 32-bit 0x100C */ - JOB_CONTROL__JOB_IRQ_JS_STATE, /* (RO) 32-bit 0x1010 */ - JOB_CONTROL__JOB_IRQ_THROTTLE, /* (RW) 32-bit 0x1014 */ - JOB_CONTROL__EVENT_IRQ_RAWSTAT, /* (RW) 32-bit 0x1020 */ - JOB_CONTROL__EVENT_IRQ_CLEAR, /* (WO) 32-bit 0x1024 */ - JOB_CONTROL__EVENT_IRQ_MASK, /* (RW) 32-bit 0x1028 */ - JOB_CONTROL__EVENT_IRQ_STATUS, /* (RO) 32-bit 0x102C */ - JOB_CONTROL__JS0__HEAD, /* (RO) 64-bit 0x1800 */ - JOB_CONTROL__JS0__TAIL, /* (RO) 64-bit 0x1808 */ - JOB_CONTROL__JS0__AFFINITY, /* (RO) 64-bit 0x1810 */ - JOB_CONTROL__JS0__CONFIG, /* (RO) 32-bit 0x1818 */ - JOB_CONTROL__JS0__XAFFINITY, /* (RO) 32-bit 0x181C */ - JOB_CONTROL__JS0__COMMAND, /* (RW) 32-bit 0x1820 */ - JOB_CONTROL__JS0__STATUS, /* (RO) 32-bit 0x1824 */ - JOB_CONTROL__JS0__FLUSH_ID, /* (RO) 32-bit 0x1830 */ - JOB_CONTROL__JS0__EVENT_0, /* (RW) 32-bit 0x1834 */ - JOB_CONTROL__JS0__EVENT_1, /* (RW) 32-bit 0x1838 */ - JOB_CONTROL__JS0__EVENT_MASK, /* (RW) 32-bit 0x183C */ - JOB_CONTROL__JS0__HEAD_NEXT, /* (RW) 64-bit 0x1840 */ - JOB_CONTROL__JS0__TAIL_NEXT, /* (RW) 64-bit 0x1848 */ - JOB_CONTROL__JS0__AFFINITY_NEXT, /* (RW) 64-bit 0x1850 */ - JOB_CONTROL__JS0__CONFIG_NEXT, /* (RW) 32-bit 0x1858 */ - JOB_CONTROL__JS0__XAFFINITY_NEXT, /* (RW) 32-bit 0x185C */ - JOB_CONTROL__JS0__COMMAND_NEXT, /* (RW) 32-bit 0x1860 */ - JOB_CONTROL__JS0__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1870 */ - JOB_CONTROL__JS0__EVENT_MASK_NEXT, /* (RW) 32-bit 0x187C */ - JOB_CONTROL__JS1__HEAD, /* (RO) 64-bit 0x1880 */ - JOB_CONTROL__JS1__TAIL, /* (RO) 64-bit 0x1888 */ - JOB_CONTROL__JS1__AFFINITY, /* (RO) 64-bit 0x1890 */ - JOB_CONTROL__JS1__CONFIG, /* (RO) 32-bit 0x1898 */ - JOB_CONTROL__JS1__XAFFINITY, /* (RO) 32-bit 0x189C */ - JOB_CONTROL__JS1__COMMAND, /* (RW) 32-bit 0x18A0 */ - JOB_CONTROL__JS1__STATUS, /* (RO) 32-bit 0x18A4 */ - JOB_CONTROL__JS1__FLUSH_ID, /* (RO) 32-bit 0x18B0 */ - JOB_CONTROL__JS1__EVENT_0, /* (RW) 32-bit 0x18B4 */ - JOB_CONTROL__JS1__EVENT_1, /* (RW) 32-bit 0x18B8 */ - JOB_CONTROL__JS1__EVENT_MASK, /* (RW) 32-bit 0x18BC */ - JOB_CONTROL__JS1__HEAD_NEXT, /* (RW) 64-bit 0x18C0 */ - JOB_CONTROL__JS1__TAIL_NEXT, /* (RW) 64-bit 0x18C8 */ - JOB_CONTROL__JS1__AFFINITY_NEXT, /* (RW) 64-bit 0x18D0 */ - JOB_CONTROL__JS1__CONFIG_NEXT, /* (RW) 32-bit 0x18D8 */ - JOB_CONTROL__JS1__XAFFINITY_NEXT, /* (RW) 32-bit 0x18DC */ - JOB_CONTROL__JS1__COMMAND_NEXT, /* (RW) 32-bit 0x18E0 */ - JOB_CONTROL__JS1__FLUSH_ID_NEXT, /* (RW) 32-bit 0x18F0 */ - JOB_CONTROL__JS1__EVENT_MASK_NEXT, /* (RW) 32-bit 0x18FC */ - JOB_CONTROL__JS2__HEAD, /* (RO) 64-bit 0x1900 */ - JOB_CONTROL__JS2__TAIL, /* (RO) 64-bit 0x1908 */ - JOB_CONTROL__JS2__AFFINITY, /* (RO) 64-bit 0x1910 */ - JOB_CONTROL__JS2__CONFIG, /* (RO) 32-bit 0x1918 */ - JOB_CONTROL__JS2__XAFFINITY, /* (RO) 32-bit 0x191C */ - JOB_CONTROL__JS2__COMMAND, /* (RW) 32-bit 0x1920 */ - JOB_CONTROL__JS2__STATUS, /* (RO) 32-bit 0x1924 */ - JOB_CONTROL__JS2__FLUSH_ID, /* (RO) 32-bit 0x1930 */ - JOB_CONTROL__JS2__EVENT_0, /* (RW) 32-bit 0x1934 */ - JOB_CONTROL__JS2__EVENT_1, /* (RW) 32-bit 0x1938 */ - JOB_CONTROL__JS2__EVENT_MASK, /* (RW) 32-bit 0x193C */ - JOB_CONTROL__JS2__HEAD_NEXT, /* (RW) 64-bit 0x1940 */ - JOB_CONTROL__JS2__TAIL_NEXT, /* (RW) 64-bit 0x1948 */ - JOB_CONTROL__JS2__AFFINITY_NEXT, /* (RW) 64-bit 0x1950 */ - JOB_CONTROL__JS2__CONFIG_NEXT, /* (RW) 32-bit 0x1958 */ - JOB_CONTROL__JS2__XAFFINITY_NEXT, /* (RW) 32-bit 0x195C */ - JOB_CONTROL__JS2__COMMAND_NEXT, /* (RW) 32-bit 0x1960 */ - JOB_CONTROL__JS2__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1970 */ - JOB_CONTROL__JS2__EVENT_MASK_NEXT, /* (RW) 32-bit 0x197C */ - JOB_CONTROL__JS3__HEAD, /* (RO) 64-bit 0x1980 */ - JOB_CONTROL__JS3__TAIL, /* (RO) 64-bit 0x1988 */ - JOB_CONTROL__JS3__AFFINITY, /* (RO) 64-bit 0x1990 */ - JOB_CONTROL__JS3__CONFIG, /* (RO) 32-bit 0x1998 */ - JOB_CONTROL__JS3__XAFFINITY, /* (RO) 32-bit 0x199C */ - JOB_CONTROL__JS3__COMMAND, /* (RW) 32-bit 0x19A0 */ - JOB_CONTROL__JS3__STATUS, /* (RO) 32-bit 0x19A4 */ - JOB_CONTROL__JS3__FLUSH_ID, /* (RO) 32-bit 0x19B0 */ - JOB_CONTROL__JS3__EVENT_0, /* (RW) 32-bit 0x19B4 */ - JOB_CONTROL__JS3__EVENT_1, /* (RW) 32-bit 0x19B8 */ - JOB_CONTROL__JS3__EVENT_MASK, /* (RW) 32-bit 0x19BC */ - JOB_CONTROL__JS3__HEAD_NEXT, /* (RW) 64-bit 0x19C0 */ - JOB_CONTROL__JS3__TAIL_NEXT, /* (RW) 64-bit 0x19C8 */ - JOB_CONTROL__JS3__AFFINITY_NEXT, /* (RW) 64-bit 0x19D0 */ - JOB_CONTROL__JS3__CONFIG_NEXT, /* (RW) 32-bit 0x19D8 */ - JOB_CONTROL__JS3__XAFFINITY_NEXT, /* (RW) 32-bit 0x19DC */ - JOB_CONTROL__JS3__COMMAND_NEXT, /* (RW) 32-bit 0x19E0 */ - JOB_CONTROL__JS3__FLUSH_ID_NEXT, /* (RW) 32-bit 0x19F0 */ - JOB_CONTROL__JS3__EVENT_MASK_NEXT, /* (RW) 32-bit 0x19FC */ - JOB_CONTROL__JS4__HEAD, /* (RO) 64-bit 0x1A00 */ - JOB_CONTROL__JS4__TAIL, /* (RO) 64-bit 0x1A08 */ - JOB_CONTROL__JS4__AFFINITY, /* (RO) 64-bit 0x1A10 */ - JOB_CONTROL__JS4__CONFIG, /* (RO) 32-bit 0x1A18 */ - JOB_CONTROL__JS4__XAFFINITY, /* (RO) 32-bit 0x1A1C */ - JOB_CONTROL__JS4__COMMAND, /* (RW) 32-bit 0x1A20 */ - JOB_CONTROL__JS4__STATUS, /* (RO) 32-bit 0x1A24 */ - JOB_CONTROL__JS4__FLUSH_ID, /* (RO) 32-bit 0x1A30 */ - JOB_CONTROL__JS4__EVENT_0, /* (RW) 32-bit 0x1A34 */ - JOB_CONTROL__JS4__EVENT_1, /* (RW) 32-bit 0x1A38 */ - JOB_CONTROL__JS4__EVENT_MASK, /* (RW) 32-bit 0x1A3C */ - JOB_CONTROL__JS4__HEAD_NEXT, /* (RW) 64-bit 0x1A40 */ - JOB_CONTROL__JS4__TAIL_NEXT, /* (RW) 64-bit 0x1A48 */ - JOB_CONTROL__JS4__AFFINITY_NEXT, /* (RW) 64-bit 0x1A50 */ - JOB_CONTROL__JS4__CONFIG_NEXT, /* (RW) 32-bit 0x1A58 */ - JOB_CONTROL__JS4__XAFFINITY_NEXT, /* (RW) 32-bit 0x1A5C */ - JOB_CONTROL__JS4__COMMAND_NEXT, /* (RW) 32-bit 0x1A60 */ - JOB_CONTROL__JS4__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1A70 */ - JOB_CONTROL__JS4__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1A7C */ - JOB_CONTROL__JS5__HEAD, /* (RO) 64-bit 0x1A80 */ - JOB_CONTROL__JS5__TAIL, /* (RO) 64-bit 0x1A88 */ - JOB_CONTROL__JS5__AFFINITY, /* (RO) 64-bit 0x1A90 */ - JOB_CONTROL__JS5__CONFIG, /* (RO) 32-bit 0x1A98 */ - JOB_CONTROL__JS5__XAFFINITY, /* (RO) 32-bit 0x1A9C */ - JOB_CONTROL__JS5__COMMAND, /* (RW) 32-bit 0x1AA0 */ - JOB_CONTROL__JS5__STATUS, /* (RO) 32-bit 0x1AA4 */ - JOB_CONTROL__JS5__FLUSH_ID, /* (RO) 32-bit 0x1AB0 */ - JOB_CONTROL__JS5__EVENT_0, /* (RW) 32-bit 0x1AB4 */ - JOB_CONTROL__JS5__EVENT_1, /* (RW) 32-bit 0x1AB8 */ - JOB_CONTROL__JS5__EVENT_MASK, /* (RW) 32-bit 0x1ABC */ - JOB_CONTROL__JS5__HEAD_NEXT, /* (RW) 64-bit 0x1AC0 */ - JOB_CONTROL__JS5__TAIL_NEXT, /* (RW) 64-bit 0x1AC8 */ - JOB_CONTROL__JS5__AFFINITY_NEXT, /* (RW) 64-bit 0x1AD0 */ - JOB_CONTROL__JS5__CONFIG_NEXT, /* (RW) 32-bit 0x1AD8 */ - JOB_CONTROL__JS5__XAFFINITY_NEXT, /* (RW) 32-bit 0x1ADC */ - JOB_CONTROL__JS5__COMMAND_NEXT, /* (RW) 32-bit 0x1AE0 */ - JOB_CONTROL__JS5__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1AF0 */ - JOB_CONTROL__JS5__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1AFC */ - JOB_CONTROL__JS6__HEAD, /* (RO) 64-bit 0x1B00 */ - JOB_CONTROL__JS6__TAIL, /* (RO) 64-bit 0x1B08 */ - JOB_CONTROL__JS6__AFFINITY, /* (RO) 64-bit 0x1B10 */ - JOB_CONTROL__JS6__CONFIG, /* (RO) 32-bit 0x1B18 */ - JOB_CONTROL__JS6__XAFFINITY, /* (RO) 32-bit 0x1B1C */ - JOB_CONTROL__JS6__COMMAND, /* (RW) 32-bit 0x1B20 */ - JOB_CONTROL__JS6__STATUS, /* (RO) 32-bit 0x1B24 */ - JOB_CONTROL__JS6__FLUSH_ID, /* (RO) 32-bit 0x1B30 */ - JOB_CONTROL__JS6__EVENT_0, /* (RW) 32-bit 0x1B34 */ - JOB_CONTROL__JS6__EVENT_1, /* (RW) 32-bit 0x1B38 */ - JOB_CONTROL__JS6__EVENT_MASK, /* (RW) 32-bit 0x1B3C */ - JOB_CONTROL__JS6__HEAD_NEXT, /* (RW) 64-bit 0x1B40 */ - JOB_CONTROL__JS6__TAIL_NEXT, /* (RW) 64-bit 0x1B48 */ - JOB_CONTROL__JS6__AFFINITY_NEXT, /* (RW) 64-bit 0x1B50 */ - JOB_CONTROL__JS6__CONFIG_NEXT, /* (RW) 32-bit 0x1B58 */ - JOB_CONTROL__JS6__XAFFINITY_NEXT, /* (RW) 32-bit 0x1B5C */ - JOB_CONTROL__JS6__COMMAND_NEXT, /* (RW) 32-bit 0x1B60 */ - JOB_CONTROL__JS6__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1B70 */ - JOB_CONTROL__JS6__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1B7C */ - JOB_CONTROL__JS7__HEAD, /* (RO) 64-bit 0x1B80 */ - JOB_CONTROL__JS7__TAIL, /* (RO) 64-bit 0x1B88 */ - JOB_CONTROL__JS7__AFFINITY, /* (RO) 64-bit 0x1B90 */ - JOB_CONTROL__JS7__CONFIG, /* (RO) 32-bit 0x1B98 */ - JOB_CONTROL__JS7__XAFFINITY, /* (RO) 32-bit 0x1B9C */ - JOB_CONTROL__JS7__COMMAND, /* (RW) 32-bit 0x1BA0 */ - JOB_CONTROL__JS7__STATUS, /* (RO) 32-bit 0x1BA4 */ - JOB_CONTROL__JS7__FLUSH_ID, /* (RO) 32-bit 0x1BB0 */ - JOB_CONTROL__JS7__EVENT_0, /* (RW) 32-bit 0x1BB4 */ - JOB_CONTROL__JS7__EVENT_1, /* (RW) 32-bit 0x1BB8 */ - JOB_CONTROL__JS7__EVENT_MASK, /* (RW) 32-bit 0x1BBC */ - JOB_CONTROL__JS7__HEAD_NEXT, /* (RW) 64-bit 0x1BC0 */ - JOB_CONTROL__JS7__TAIL_NEXT, /* (RW) 64-bit 0x1BC8 */ - JOB_CONTROL__JS7__AFFINITY_NEXT, /* (RW) 64-bit 0x1BD0 */ - JOB_CONTROL__JS7__CONFIG_NEXT, /* (RW) 32-bit 0x1BD8 */ - JOB_CONTROL__JS7__XAFFINITY_NEXT, /* (RW) 32-bit 0x1BDC */ - JOB_CONTROL__JS7__COMMAND_NEXT, /* (RW) 32-bit 0x1BE0 */ - JOB_CONTROL__JS7__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1BF0 */ - JOB_CONTROL__JS7__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1BFC */ - JOB_CONTROL__JS8__HEAD, /* (RO) 64-bit 0x1C00 */ - JOB_CONTROL__JS8__TAIL, /* (RO) 64-bit 0x1C08 */ - JOB_CONTROL__JS8__AFFINITY, /* (RO) 64-bit 0x1C10 */ - JOB_CONTROL__JS8__CONFIG, /* (RO) 32-bit 0x1C18 */ - JOB_CONTROL__JS8__XAFFINITY, /* (RO) 32-bit 0x1C1C */ - JOB_CONTROL__JS8__COMMAND, /* (RW) 32-bit 0x1C20 */ - JOB_CONTROL__JS8__STATUS, /* (RO) 32-bit 0x1C24 */ - JOB_CONTROL__JS8__FLUSH_ID, /* (RO) 32-bit 0x1C30 */ - JOB_CONTROL__JS8__EVENT_0, /* (RW) 32-bit 0x1C34 */ - JOB_CONTROL__JS8__EVENT_1, /* (RW) 32-bit 0x1C38 */ - JOB_CONTROL__JS8__EVENT_MASK, /* (RW) 32-bit 0x1C3C */ - JOB_CONTROL__JS8__HEAD_NEXT, /* (RW) 64-bit 0x1C40 */ - JOB_CONTROL__JS8__TAIL_NEXT, /* (RW) 64-bit 0x1C48 */ - JOB_CONTROL__JS8__AFFINITY_NEXT, /* (RW) 64-bit 0x1C50 */ - JOB_CONTROL__JS8__CONFIG_NEXT, /* (RW) 32-bit 0x1C58 */ - JOB_CONTROL__JS8__XAFFINITY_NEXT, /* (RW) 32-bit 0x1C5C */ - JOB_CONTROL__JS8__COMMAND_NEXT, /* (RW) 32-bit 0x1C60 */ - JOB_CONTROL__JS8__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1C70 */ - JOB_CONTROL__JS8__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1C7C */ - JOB_CONTROL__JS9__HEAD, /* (RO) 64-bit 0x1C80 */ - JOB_CONTROL__JS9__TAIL, /* (RO) 64-bit 0x1C88 */ - JOB_CONTROL__JS9__AFFINITY, /* (RO) 64-bit 0x1C90 */ - JOB_CONTROL__JS9__CONFIG, /* (RO) 32-bit 0x1C98 */ - JOB_CONTROL__JS9__XAFFINITY, /* (RO) 32-bit 0x1C9C */ - JOB_CONTROL__JS9__COMMAND, /* (RW) 32-bit 0x1CA0 */ - JOB_CONTROL__JS9__STATUS, /* (RO) 32-bit 0x1CA4 */ - JOB_CONTROL__JS9__FLUSH_ID, /* (RO) 32-bit 0x1CB0 */ - JOB_CONTROL__JS9__EVENT_0, /* (RW) 32-bit 0x1CB4 */ - JOB_CONTROL__JS9__EVENT_1, /* (RW) 32-bit 0x1CB8 */ - JOB_CONTROL__JS9__EVENT_MASK, /* (RW) 32-bit 0x1CBC */ - JOB_CONTROL__JS9__HEAD_NEXT, /* (RW) 64-bit 0x1CC0 */ - JOB_CONTROL__JS9__TAIL_NEXT, /* (RW) 64-bit 0x1CC8 */ - JOB_CONTROL__JS9__AFFINITY_NEXT, /* (RW) 64-bit 0x1CD0 */ - JOB_CONTROL__JS9__CONFIG_NEXT, /* (RW) 32-bit 0x1CD8 */ - JOB_CONTROL__JS9__XAFFINITY_NEXT, /* (RW) 32-bit 0x1CDC */ - JOB_CONTROL__JS9__COMMAND_NEXT, /* (RW) 32-bit 0x1CE0 */ - JOB_CONTROL__JS9__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1CF0 */ - JOB_CONTROL__JS9__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1CFC */ - JOB_CONTROL__JS10__HEAD, /* (RO) 64-bit 0x1D00 */ - JOB_CONTROL__JS10__TAIL, /* (RO) 64-bit 0x1D08 */ - JOB_CONTROL__JS10__AFFINITY, /* (RO) 64-bit 0x1D10 */ - JOB_CONTROL__JS10__CONFIG, /* (RO) 32-bit 0x1D18 */ - JOB_CONTROL__JS10__XAFFINITY, /* (RO) 32-bit 0x1D1C */ - JOB_CONTROL__JS10__COMMAND, /* (RW) 32-bit 0x1D20 */ - JOB_CONTROL__JS10__STATUS, /* (RO) 32-bit 0x1D24 */ - JOB_CONTROL__JS10__FLUSH_ID, /* (RO) 32-bit 0x1D30 */ - JOB_CONTROL__JS10__EVENT_0, /* (RW) 32-bit 0x1D34 */ - JOB_CONTROL__JS10__EVENT_1, /* (RW) 32-bit 0x1D38 */ - JOB_CONTROL__JS10__EVENT_MASK, /* (RW) 32-bit 0x1D3C */ - JOB_CONTROL__JS10__HEAD_NEXT, /* (RW) 64-bit 0x1D40 */ - JOB_CONTROL__JS10__TAIL_NEXT, /* (RW) 64-bit 0x1D48 */ - JOB_CONTROL__JS10__AFFINITY_NEXT, /* (RW) 64-bit 0x1D50 */ - JOB_CONTROL__JS10__CONFIG_NEXT, /* (RW) 32-bit 0x1D58 */ - JOB_CONTROL__JS10__XAFFINITY_NEXT, /* (RW) 32-bit 0x1D5C */ - JOB_CONTROL__JS10__COMMAND_NEXT, /* (RW) 32-bit 0x1D60 */ - JOB_CONTROL__JS10__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1D70 */ - JOB_CONTROL__JS10__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1D7C */ - JOB_CONTROL__JS11__HEAD, /* (RO) 64-bit 0x1D80 */ - JOB_CONTROL__JS11__TAIL, /* (RO) 64-bit 0x1D88 */ - JOB_CONTROL__JS11__AFFINITY, /* (RO) 64-bit 0x1D90 */ - JOB_CONTROL__JS11__CONFIG, /* (RO) 32-bit 0x1D98 */ - JOB_CONTROL__JS11__XAFFINITY, /* (RO) 32-bit 0x1D9C */ - JOB_CONTROL__JS11__COMMAND, /* (RW) 32-bit 0x1DA0 */ - JOB_CONTROL__JS11__STATUS, /* (RO) 32-bit 0x1DA4 */ - JOB_CONTROL__JS11__FLUSH_ID, /* (RO) 32-bit 0x1DB0 */ - JOB_CONTROL__JS11__EVENT_0, /* (RW) 32-bit 0x1DB4 */ - JOB_CONTROL__JS11__EVENT_1, /* (RW) 32-bit 0x1DB8 */ - JOB_CONTROL__JS11__EVENT_MASK, /* (RW) 32-bit 0x1DBC */ - JOB_CONTROL__JS11__HEAD_NEXT, /* (RW) 64-bit 0x1DC0 */ - JOB_CONTROL__JS11__TAIL_NEXT, /* (RW) 64-bit 0x1DC8 */ - JOB_CONTROL__JS11__AFFINITY_NEXT, /* (RW) 64-bit 0x1DD0 */ - JOB_CONTROL__JS11__CONFIG_NEXT, /* (RW) 32-bit 0x1DD8 */ - JOB_CONTROL__JS11__XAFFINITY_NEXT, /* (RW) 32-bit 0x1DDC */ - JOB_CONTROL__JS11__COMMAND_NEXT, /* (RW) 32-bit 0x1DE0 */ - JOB_CONTROL__JS11__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1DF0 */ - JOB_CONTROL__JS11__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1DFC */ - JOB_CONTROL__JS12__HEAD, /* (RO) 64-bit 0x1E00 */ - JOB_CONTROL__JS12__TAIL, /* (RO) 64-bit 0x1E08 */ - JOB_CONTROL__JS12__AFFINITY, /* (RO) 64-bit 0x1E10 */ - JOB_CONTROL__JS12__CONFIG, /* (RO) 32-bit 0x1E18 */ - JOB_CONTROL__JS12__XAFFINITY, /* (RO) 32-bit 0x1E1C */ - JOB_CONTROL__JS12__COMMAND, /* (RW) 32-bit 0x1E20 */ - JOB_CONTROL__JS12__STATUS, /* (RO) 32-bit 0x1E24 */ - JOB_CONTROL__JS12__FLUSH_ID, /* (RO) 32-bit 0x1E30 */ - JOB_CONTROL__JS12__EVENT_0, /* (RW) 32-bit 0x1E34 */ - JOB_CONTROL__JS12__EVENT_1, /* (RW) 32-bit 0x1E38 */ - JOB_CONTROL__JS12__EVENT_MASK, /* (RW) 32-bit 0x1E3C */ - JOB_CONTROL__JS12__HEAD_NEXT, /* (RW) 64-bit 0x1E40 */ - JOB_CONTROL__JS12__TAIL_NEXT, /* (RW) 64-bit 0x1E48 */ - JOB_CONTROL__JS12__AFFINITY_NEXT, /* (RW) 64-bit 0x1E50 */ - JOB_CONTROL__JS12__CONFIG_NEXT, /* (RW) 32-bit 0x1E58 */ - JOB_CONTROL__JS12__XAFFINITY_NEXT, /* (RW) 32-bit 0x1E5C */ - JOB_CONTROL__JS12__COMMAND_NEXT, /* (RW) 32-bit 0x1E60 */ - JOB_CONTROL__JS12__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1E70 */ - JOB_CONTROL__JS12__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1E7C */ - JOB_CONTROL__JS13__HEAD, /* (RO) 64-bit 0x1E80 */ - JOB_CONTROL__JS13__TAIL, /* (RO) 64-bit 0x1E88 */ - JOB_CONTROL__JS13__AFFINITY, /* (RO) 64-bit 0x1E90 */ - JOB_CONTROL__JS13__CONFIG, /* (RO) 32-bit 0x1E98 */ - JOB_CONTROL__JS13__XAFFINITY, /* (RO) 32-bit 0x1E9C */ - JOB_CONTROL__JS13__COMMAND, /* (RW) 32-bit 0x1EA0 */ - JOB_CONTROL__JS13__STATUS, /* (RO) 32-bit 0x1EA4 */ - JOB_CONTROL__JS13__FLUSH_ID, /* (RO) 32-bit 0x1EB0 */ - JOB_CONTROL__JS13__EVENT_0, /* (RW) 32-bit 0x1EB4 */ - JOB_CONTROL__JS13__EVENT_1, /* (RW) 32-bit 0x1EB8 */ - JOB_CONTROL__JS13__EVENT_MASK, /* (RW) 32-bit 0x1EBC */ - JOB_CONTROL__JS13__HEAD_NEXT, /* (RW) 64-bit 0x1EC0 */ - JOB_CONTROL__JS13__TAIL_NEXT, /* (RW) 64-bit 0x1EC8 */ - JOB_CONTROL__JS13__AFFINITY_NEXT, /* (RW) 64-bit 0x1ED0 */ - JOB_CONTROL__JS13__CONFIG_NEXT, /* (RW) 32-bit 0x1ED8 */ - JOB_CONTROL__JS13__XAFFINITY_NEXT, /* (RW) 32-bit 0x1EDC */ - JOB_CONTROL__JS13__COMMAND_NEXT, /* (RW) 32-bit 0x1EE0 */ - JOB_CONTROL__JS13__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1EF0 */ - JOB_CONTROL__JS13__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1EFC */ - JOB_CONTROL__JS14__HEAD, /* (RO) 64-bit 0x1F00 */ - JOB_CONTROL__JS14__TAIL, /* (RO) 64-bit 0x1F08 */ - JOB_CONTROL__JS14__AFFINITY, /* (RO) 64-bit 0x1F10 */ - JOB_CONTROL__JS14__CONFIG, /* (RO) 32-bit 0x1F18 */ - JOB_CONTROL__JS14__XAFFINITY, /* (RO) 32-bit 0x1F1C */ - JOB_CONTROL__JS14__COMMAND, /* (RW) 32-bit 0x1F20 */ - JOB_CONTROL__JS14__STATUS, /* (RO) 32-bit 0x1F24 */ - JOB_CONTROL__JS14__FLUSH_ID, /* (RO) 32-bit 0x1F30 */ - JOB_CONTROL__JS14__EVENT_0, /* (RW) 32-bit 0x1F34 */ - JOB_CONTROL__JS14__EVENT_1, /* (RW) 32-bit 0x1F38 */ - JOB_CONTROL__JS14__EVENT_MASK, /* (RW) 32-bit 0x1F3C */ - JOB_CONTROL__JS14__HEAD_NEXT, /* (RW) 64-bit 0x1F40 */ - JOB_CONTROL__JS14__TAIL_NEXT, /* (RW) 64-bit 0x1F48 */ - JOB_CONTROL__JS14__AFFINITY_NEXT, /* (RW) 64-bit 0x1F50 */ - JOB_CONTROL__JS14__CONFIG_NEXT, /* (RW) 32-bit 0x1F58 */ - JOB_CONTROL__JS14__XAFFINITY_NEXT, /* (RW) 32-bit 0x1F5C */ - JOB_CONTROL__JS14__COMMAND_NEXT, /* (RW) 32-bit 0x1F60 */ - JOB_CONTROL__JS14__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1F70 */ - JOB_CONTROL__JS14__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1F7C */ - JOB_CONTROL__JS15__HEAD, /* (RO) 64-bit 0x1F80 */ - JOB_CONTROL__JS15__TAIL, /* (RO) 64-bit 0x1F88 */ - JOB_CONTROL__JS15__AFFINITY, /* (RO) 64-bit 0x1F90 */ - JOB_CONTROL__JS15__CONFIG, /* (RO) 32-bit 0x1F98 */ - JOB_CONTROL__JS15__XAFFINITY, /* (RO) 32-bit 0x1F9C */ - JOB_CONTROL__JS15__COMMAND, /* (RW) 32-bit 0x1FA0 */ - JOB_CONTROL__JS15__STATUS, /* (RO) 32-bit 0x1FA4 */ - JOB_CONTROL__JS15__FLUSH_ID, /* (RO) 32-bit 0x1FB0 */ - JOB_CONTROL__JS15__EVENT_0, /* (RW) 32-bit 0x1FB4 */ - JOB_CONTROL__JS15__EVENT_1, /* (RW) 32-bit 0x1FB8 */ - JOB_CONTROL__JS15__EVENT_MASK, /* (RW) 32-bit 0x1FBC */ - JOB_CONTROL__JS15__HEAD_NEXT, /* (RW) 64-bit 0x1FC0 */ - JOB_CONTROL__JS15__TAIL_NEXT, /* (RW) 64-bit 0x1FC8 */ - JOB_CONTROL__JS15__AFFINITY_NEXT, /* (RW) 64-bit 0x1FD0 */ - JOB_CONTROL__JS15__CONFIG_NEXT, /* (RW) 32-bit 0x1FD8 */ - JOB_CONTROL__JS15__XAFFINITY_NEXT, /* (RW) 32-bit 0x1FDC */ - JOB_CONTROL__JS15__COMMAND_NEXT, /* (RW) 32-bit 0x1FE0 */ - JOB_CONTROL__JS15__FLUSH_ID_NEXT, /* (RW) 32-bit 0x1FF0 */ - JOB_CONTROL__JS15__EVENT_MASK_NEXT, /* (RW) 32-bit 0x1FFC */ - MMU_STAGE1__ST1MMU__IRQ_RAWSTAT, /* (RW) 32-bit 0x2000 */ - MMU_STAGE1__ST1MMU__IRQ_CLEAR, /* (WO) 32-bit 0x2004 */ - MMU_STAGE1__ST1MMU__IRQ_MASK, /* (RW) 32-bit 0x2008 */ - MMU_STAGE1__ST1MMU__IRQ_STATUS, /* (RO) 32-bit 0x200C */ - MMU_STAGE1__ST1MMU__AS0__TRANSTAB, /* (RW) 64-bit 0x2400 */ - MMU_STAGE1__ST1MMU__AS0__MEMATTR, /* (RW) 64-bit 0x2408 */ - MMU_STAGE1__ST1MMU__AS0__LOCKADDR, /* (RW) 64-bit 0x2410 */ - MMU_STAGE1__ST1MMU__AS0__COMMAND, /* (WO) 32-bit 0x2418 */ - MMU_STAGE1__ST1MMU__AS0__FAULTSTATUS, /* (RO) 32-bit 0x241C */ - MMU_STAGE1__ST1MMU__AS0__FAULTADDRESS, /* (RO) 64-bit 0x2420 */ - MMU_STAGE1__ST1MMU__AS0__STATUS, /* (RO) 32-bit 0x2428 */ - MMU_STAGE1__ST1MMU__AS0__TRANSCFG, /* (RW) 64-bit 0x2430 */ - MMU_STAGE1__ST1MMU__AS0__FAULTEXTRA, /* (RO) 64-bit 0x2438 */ - MMU_STAGE1__ST1MMU__AS1__TRANSTAB, /* (RW) 64-bit 0x2440 */ - MMU_STAGE1__ST1MMU__AS1__MEMATTR, /* (RW) 64-bit 0x2448 */ - MMU_STAGE1__ST1MMU__AS1__LOCKADDR, /* (RW) 64-bit 0x2450 */ - MMU_STAGE1__ST1MMU__AS1__COMMAND, /* (WO) 32-bit 0x2458 */ - MMU_STAGE1__ST1MMU__AS1__FAULTSTATUS, /* (RO) 32-bit 0x245C */ - MMU_STAGE1__ST1MMU__AS1__FAULTADDRESS, /* (RO) 64-bit 0x2460 */ - MMU_STAGE1__ST1MMU__AS1__STATUS, /* (RO) 32-bit 0x2468 */ - MMU_STAGE1__ST1MMU__AS1__TRANSCFG, /* (RW) 64-bit 0x2470 */ - MMU_STAGE1__ST1MMU__AS1__FAULTEXTRA, /* (RO) 64-bit 0x2478 */ - MMU_STAGE1__ST1MMU__AS2__TRANSTAB, /* (RW) 64-bit 0x2480 */ - MMU_STAGE1__ST1MMU__AS2__MEMATTR, /* (RW) 64-bit 0x2488 */ - MMU_STAGE1__ST1MMU__AS2__LOCKADDR, /* (RW) 64-bit 0x2490 */ - MMU_STAGE1__ST1MMU__AS2__COMMAND, /* (WO) 32-bit 0x2498 */ - MMU_STAGE1__ST1MMU__AS2__FAULTSTATUS, /* (RO) 32-bit 0x249C */ - MMU_STAGE1__ST1MMU__AS2__FAULTADDRESS, /* (RO) 64-bit 0x24A0 */ - MMU_STAGE1__ST1MMU__AS2__STATUS, /* (RO) 32-bit 0x24A8 */ - MMU_STAGE1__ST1MMU__AS2__TRANSCFG, /* (RW) 64-bit 0x24B0 */ - MMU_STAGE1__ST1MMU__AS2__FAULTEXTRA, /* (RO) 64-bit 0x24B8 */ - MMU_STAGE1__ST1MMU__AS3__TRANSTAB, /* (RW) 64-bit 0x24C0 */ - MMU_STAGE1__ST1MMU__AS3__MEMATTR, /* (RW) 64-bit 0x24C8 */ - MMU_STAGE1__ST1MMU__AS3__LOCKADDR, /* (RW) 64-bit 0x24D0 */ - MMU_STAGE1__ST1MMU__AS3__COMMAND, /* (WO) 32-bit 0x24D8 */ - MMU_STAGE1__ST1MMU__AS3__FAULTSTATUS, /* (RO) 32-bit 0x24DC */ - MMU_STAGE1__ST1MMU__AS3__FAULTADDRESS, /* (RO) 64-bit 0x24E0 */ - MMU_STAGE1__ST1MMU__AS3__STATUS, /* (RO) 32-bit 0x24E8 */ - MMU_STAGE1__ST1MMU__AS3__TRANSCFG, /* (RW) 64-bit 0x24F0 */ - MMU_STAGE1__ST1MMU__AS3__FAULTEXTRA, /* (RO) 64-bit 0x24F8 */ - MMU_STAGE1__ST1MMU__AS4__TRANSTAB, /* (RW) 64-bit 0x2500 */ - MMU_STAGE1__ST1MMU__AS4__MEMATTR, /* (RW) 64-bit 0x2508 */ - MMU_STAGE1__ST1MMU__AS4__LOCKADDR, /* (RW) 64-bit 0x2510 */ - MMU_STAGE1__ST1MMU__AS4__COMMAND, /* (WO) 32-bit 0x2518 */ - MMU_STAGE1__ST1MMU__AS4__FAULTSTATUS, /* (RO) 32-bit 0x251C */ - MMU_STAGE1__ST1MMU__AS4__FAULTADDRESS, /* (RO) 64-bit 0x2520 */ - MMU_STAGE1__ST1MMU__AS4__STATUS, /* (RO) 32-bit 0x2528 */ - MMU_STAGE1__ST1MMU__AS4__TRANSCFG, /* (RW) 64-bit 0x2530 */ - MMU_STAGE1__ST1MMU__AS4__FAULTEXTRA, /* (RO) 64-bit 0x2538 */ - MMU_STAGE1__ST1MMU__AS5__TRANSTAB, /* (RW) 64-bit 0x2540 */ - MMU_STAGE1__ST1MMU__AS5__MEMATTR, /* (RW) 64-bit 0x2548 */ - MMU_STAGE1__ST1MMU__AS5__LOCKADDR, /* (RW) 64-bit 0x2550 */ - MMU_STAGE1__ST1MMU__AS5__COMMAND, /* (WO) 32-bit 0x2558 */ - MMU_STAGE1__ST1MMU__AS5__FAULTSTATUS, /* (RO) 32-bit 0x255C */ - MMU_STAGE1__ST1MMU__AS5__FAULTADDRESS, /* (RO) 64-bit 0x2560 */ - MMU_STAGE1__ST1MMU__AS5__STATUS, /* (RO) 32-bit 0x2568 */ - MMU_STAGE1__ST1MMU__AS5__TRANSCFG, /* (RW) 64-bit 0x2570 */ - MMU_STAGE1__ST1MMU__AS5__FAULTEXTRA, /* (RO) 64-bit 0x2578 */ - MMU_STAGE1__ST1MMU__AS6__TRANSTAB, /* (RW) 64-bit 0x2580 */ - MMU_STAGE1__ST1MMU__AS6__MEMATTR, /* (RW) 64-bit 0x2588 */ - MMU_STAGE1__ST1MMU__AS6__LOCKADDR, /* (RW) 64-bit 0x2590 */ - MMU_STAGE1__ST1MMU__AS6__COMMAND, /* (WO) 32-bit 0x2598 */ - MMU_STAGE1__ST1MMU__AS6__FAULTSTATUS, /* (RO) 32-bit 0x259C */ - MMU_STAGE1__ST1MMU__AS6__FAULTADDRESS, /* (RO) 64-bit 0x25A0 */ - MMU_STAGE1__ST1MMU__AS6__STATUS, /* (RO) 32-bit 0x25A8 */ - MMU_STAGE1__ST1MMU__AS6__TRANSCFG, /* (RW) 64-bit 0x25B0 */ - MMU_STAGE1__ST1MMU__AS6__FAULTEXTRA, /* (RO) 64-bit 0x25B8 */ - MMU_STAGE1__ST1MMU__AS7__TRANSTAB, /* (RW) 64-bit 0x25C0 */ - MMU_STAGE1__ST1MMU__AS7__MEMATTR, /* (RW) 64-bit 0x25C8 */ - MMU_STAGE1__ST1MMU__AS7__LOCKADDR, /* (RW) 64-bit 0x25D0 */ - MMU_STAGE1__ST1MMU__AS7__COMMAND, /* (WO) 32-bit 0x25D8 */ - MMU_STAGE1__ST1MMU__AS7__FAULTSTATUS, /* (RO) 32-bit 0x25DC */ - MMU_STAGE1__ST1MMU__AS7__FAULTADDRESS, /* (RO) 64-bit 0x25E0 */ - MMU_STAGE1__ST1MMU__AS7__STATUS, /* (RO) 32-bit 0x25E8 */ - MMU_STAGE1__ST1MMU__AS7__TRANSCFG, /* (RW) 64-bit 0x25F0 */ - MMU_STAGE1__ST1MMU__AS7__FAULTEXTRA, /* (RO) 64-bit 0x25F8 */ - MMU_STAGE1__ST1MMU__AS8__TRANSTAB, /* (RW) 64-bit 0x2600 */ - MMU_STAGE1__ST1MMU__AS8__MEMATTR, /* (RW) 64-bit 0x2608 */ - MMU_STAGE1__ST1MMU__AS8__LOCKADDR, /* (RW) 64-bit 0x2610 */ - MMU_STAGE1__ST1MMU__AS8__COMMAND, /* (WO) 32-bit 0x2618 */ - MMU_STAGE1__ST1MMU__AS8__FAULTSTATUS, /* (RO) 32-bit 0x261C */ - MMU_STAGE1__ST1MMU__AS8__FAULTADDRESS, /* (RO) 64-bit 0x2620 */ - MMU_STAGE1__ST1MMU__AS8__STATUS, /* (RO) 32-bit 0x2628 */ - MMU_STAGE1__ST1MMU__AS8__TRANSCFG, /* (RW) 64-bit 0x2630 */ - MMU_STAGE1__ST1MMU__AS8__FAULTEXTRA, /* (RO) 64-bit 0x2638 */ - MMU_STAGE1__ST1MMU__AS9__TRANSTAB, /* (RW) 64-bit 0x2640 */ - MMU_STAGE1__ST1MMU__AS9__MEMATTR, /* (RW) 64-bit 0x2648 */ - MMU_STAGE1__ST1MMU__AS9__LOCKADDR, /* (RW) 64-bit 0x2650 */ - MMU_STAGE1__ST1MMU__AS9__COMMAND, /* (WO) 32-bit 0x2658 */ - MMU_STAGE1__ST1MMU__AS9__FAULTSTATUS, /* (RO) 32-bit 0x265C */ - MMU_STAGE1__ST1MMU__AS9__FAULTADDRESS, /* (RO) 64-bit 0x2660 */ - MMU_STAGE1__ST1MMU__AS9__STATUS, /* (RO) 32-bit 0x2668 */ - MMU_STAGE1__ST1MMU__AS9__TRANSCFG, /* (RW) 64-bit 0x2670 */ - MMU_STAGE1__ST1MMU__AS9__FAULTEXTRA, /* (RO) 64-bit 0x2678 */ - MMU_STAGE1__ST1MMU__AS10__TRANSTAB, /* (RW) 64-bit 0x2680 */ - MMU_STAGE1__ST1MMU__AS10__MEMATTR, /* (RW) 64-bit 0x2688 */ - MMU_STAGE1__ST1MMU__AS10__LOCKADDR, /* (RW) 64-bit 0x2690 */ - MMU_STAGE1__ST1MMU__AS10__COMMAND, /* (WO) 32-bit 0x2698 */ - MMU_STAGE1__ST1MMU__AS10__FAULTSTATUS, /* (RO) 32-bit 0x269C */ - MMU_STAGE1__ST1MMU__AS10__FAULTADDRESS, /* (RO) 64-bit 0x26A0 */ - MMU_STAGE1__ST1MMU__AS10__STATUS, /* (RO) 32-bit 0x26A8 */ - MMU_STAGE1__ST1MMU__AS10__TRANSCFG, /* (RW) 64-bit 0x26B0 */ - MMU_STAGE1__ST1MMU__AS10__FAULTEXTRA, /* (RO) 64-bit 0x26B8 */ - MMU_STAGE1__ST1MMU__AS11__TRANSTAB, /* (RW) 64-bit 0x26C0 */ - MMU_STAGE1__ST1MMU__AS11__MEMATTR, /* (RW) 64-bit 0x26C8 */ - MMU_STAGE1__ST1MMU__AS11__LOCKADDR, /* (RW) 64-bit 0x26D0 */ - MMU_STAGE1__ST1MMU__AS11__COMMAND, /* (WO) 32-bit 0x26D8 */ - MMU_STAGE1__ST1MMU__AS11__FAULTSTATUS, /* (RO) 32-bit 0x26DC */ - MMU_STAGE1__ST1MMU__AS11__FAULTADDRESS, /* (RO) 64-bit 0x26E0 */ - MMU_STAGE1__ST1MMU__AS11__STATUS, /* (RO) 32-bit 0x26E8 */ - MMU_STAGE1__ST1MMU__AS11__TRANSCFG, /* (RW) 64-bit 0x26F0 */ - MMU_STAGE1__ST1MMU__AS11__FAULTEXTRA, /* (RO) 64-bit 0x26F8 */ - MMU_STAGE1__ST1MMU__AS12__TRANSTAB, /* (RW) 64-bit 0x2700 */ - MMU_STAGE1__ST1MMU__AS12__MEMATTR, /* (RW) 64-bit 0x2708 */ - MMU_STAGE1__ST1MMU__AS12__LOCKADDR, /* (RW) 64-bit 0x2710 */ - MMU_STAGE1__ST1MMU__AS12__COMMAND, /* (WO) 32-bit 0x2718 */ - MMU_STAGE1__ST1MMU__AS12__FAULTSTATUS, /* (RO) 32-bit 0x271C */ - MMU_STAGE1__ST1MMU__AS12__FAULTADDRESS, /* (RO) 64-bit 0x2720 */ - MMU_STAGE1__ST1MMU__AS12__STATUS, /* (RO) 32-bit 0x2728 */ - MMU_STAGE1__ST1MMU__AS12__TRANSCFG, /* (RW) 64-bit 0x2730 */ - MMU_STAGE1__ST1MMU__AS12__FAULTEXTRA, /* (RO) 64-bit 0x2738 */ - MMU_STAGE1__ST1MMU__AS13__TRANSTAB, /* (RW) 64-bit 0x2740 */ - MMU_STAGE1__ST1MMU__AS13__MEMATTR, /* (RW) 64-bit 0x2748 */ - MMU_STAGE1__ST1MMU__AS13__LOCKADDR, /* (RW) 64-bit 0x2750 */ - MMU_STAGE1__ST1MMU__AS13__COMMAND, /* (WO) 32-bit 0x2758 */ - MMU_STAGE1__ST1MMU__AS13__FAULTSTATUS, /* (RO) 32-bit 0x275C */ - MMU_STAGE1__ST1MMU__AS13__FAULTADDRESS, /* (RO) 64-bit 0x2760 */ - MMU_STAGE1__ST1MMU__AS13__STATUS, /* (RO) 32-bit 0x2768 */ - MMU_STAGE1__ST1MMU__AS13__TRANSCFG, /* (RW) 64-bit 0x2770 */ - MMU_STAGE1__ST1MMU__AS13__FAULTEXTRA, /* (RO) 64-bit 0x2778 */ - MMU_STAGE1__ST1MMU__AS14__TRANSTAB, /* (RW) 64-bit 0x2780 */ - MMU_STAGE1__ST1MMU__AS14__MEMATTR, /* (RW) 64-bit 0x2788 */ - MMU_STAGE1__ST1MMU__AS14__LOCKADDR, /* (RW) 64-bit 0x2790 */ - MMU_STAGE1__ST1MMU__AS14__COMMAND, /* (WO) 32-bit 0x2798 */ - MMU_STAGE1__ST1MMU__AS14__FAULTSTATUS, /* (RO) 32-bit 0x279C */ - MMU_STAGE1__ST1MMU__AS14__FAULTADDRESS, /* (RO) 64-bit 0x27A0 */ - MMU_STAGE1__ST1MMU__AS14__STATUS, /* (RO) 32-bit 0x27A8 */ - MMU_STAGE1__ST1MMU__AS14__TRANSCFG, /* (RW) 64-bit 0x27B0 */ - MMU_STAGE1__ST1MMU__AS14__FAULTEXTRA, /* (RO) 64-bit 0x27B8 */ - MMU_STAGE1__ST1MMU__AS15__TRANSTAB, /* (RW) 64-bit 0x27C0 */ - MMU_STAGE1__ST1MMU__AS15__MEMATTR, /* (RW) 64-bit 0x27C8 */ - MMU_STAGE1__ST1MMU__AS15__LOCKADDR, /* (RW) 64-bit 0x27D0 */ - MMU_STAGE1__ST1MMU__AS15__COMMAND, /* (WO) 32-bit 0x27D8 */ - MMU_STAGE1__ST1MMU__AS15__FAULTSTATUS, /* (RO) 32-bit 0x27DC */ - MMU_STAGE1__ST1MMU__AS15__FAULTADDRESS, /* (RO) 64-bit 0x27E0 */ - MMU_STAGE1__ST1MMU__AS15__STATUS, /* (RO) 32-bit 0x27E8 */ - MMU_STAGE1__ST1MMU__AS15__TRANSCFG, /* (RW) 64-bit 0x27F0 */ - MMU_STAGE1__ST1MMU__AS15__FAULTEXTRA, /* (RO) 64-bit 0x27F8 */ - MMU_STAGE2__ST2MMU__IRQ_RAWSTAT, /* (RW) 32-bit 0x10000 */ - MMU_STAGE2__ST2MMU__IRQ_CLEAR, /* (WO) 32-bit 0x10004 */ - MMU_STAGE2__ST2MMU__IRQ_MASK, /* (RW) 32-bit 0x10008 */ - MMU_STAGE2__ST2MMU__IRQ_STATUS, /* (RO) 32-bit 0x1000C */ - MMU_STAGE2__ST2MMU__AS0__TRANSTAB, /* (RW) 64-bit 0x10400 */ - MMU_STAGE2__ST2MMU__AS0__MEMATTR, /* (RW) 64-bit 0x10408 */ - MMU_STAGE2__ST2MMU__AS0__LOCKADDR, /* (RW) 64-bit 0x10410 */ - MMU_STAGE2__ST2MMU__AS0__COMMAND, /* (WO) 32-bit 0x10418 */ - MMU_STAGE2__ST2MMU__AS0__FAULTSTATUS, /* (RO) 32-bit 0x1041C */ - MMU_STAGE2__ST2MMU__AS0__FAULTADDRESS, /* (RO) 64-bit 0x10420 */ - MMU_STAGE2__ST2MMU__AS0__STATUS, /* (RO) 32-bit 0x10428 */ - MMU_STAGE2__ST2MMU__AS0__TRANSCFG, /* (RW) 64-bit 0x10430 */ - MMU_STAGE2__ST2MMU__AS0__FAULTEXTRA, /* (RO) 64-bit 0x10438 */ - MMU_STAGE2__ST2MMU__AS1__TRANSTAB, /* (RW) 64-bit 0x10440 */ - MMU_STAGE2__ST2MMU__AS1__MEMATTR, /* (RW) 64-bit 0x10448 */ - MMU_STAGE2__ST2MMU__AS1__LOCKADDR, /* (RW) 64-bit 0x10450 */ - MMU_STAGE2__ST2MMU__AS1__COMMAND, /* (WO) 32-bit 0x10458 */ - MMU_STAGE2__ST2MMU__AS1__FAULTSTATUS, /* (RO) 32-bit 0x1045C */ - MMU_STAGE2__ST2MMU__AS1__FAULTADDRESS, /* (RO) 64-bit 0x10460 */ - MMU_STAGE2__ST2MMU__AS1__STATUS, /* (RO) 32-bit 0x10468 */ - MMU_STAGE2__ST2MMU__AS1__TRANSCFG, /* (RW) 64-bit 0x10470 */ - MMU_STAGE2__ST2MMU__AS1__FAULTEXTRA, /* (RO) 64-bit 0x10478 */ - MMU_STAGE2__ST2MMU__AS2__TRANSTAB, /* (RW) 64-bit 0x10480 */ - MMU_STAGE2__ST2MMU__AS2__MEMATTR, /* (RW) 64-bit 0x10488 */ - MMU_STAGE2__ST2MMU__AS2__LOCKADDR, /* (RW) 64-bit 0x10490 */ - MMU_STAGE2__ST2MMU__AS2__COMMAND, /* (WO) 32-bit 0x10498 */ - MMU_STAGE2__ST2MMU__AS2__FAULTSTATUS, /* (RO) 32-bit 0x1049C */ - MMU_STAGE2__ST2MMU__AS2__FAULTADDRESS, /* (RO) 64-bit 0x104A0 */ - MMU_STAGE2__ST2MMU__AS2__STATUS, /* (RO) 32-bit 0x104A8 */ - MMU_STAGE2__ST2MMU__AS2__TRANSCFG, /* (RW) 64-bit 0x104B0 */ - MMU_STAGE2__ST2MMU__AS2__FAULTEXTRA, /* (RO) 64-bit 0x104B8 */ - MMU_STAGE2__ST2MMU__AS3__TRANSTAB, /* (RW) 64-bit 0x104C0 */ - MMU_STAGE2__ST2MMU__AS3__MEMATTR, /* (RW) 64-bit 0x104C8 */ - MMU_STAGE2__ST2MMU__AS3__LOCKADDR, /* (RW) 64-bit 0x104D0 */ - MMU_STAGE2__ST2MMU__AS3__COMMAND, /* (WO) 32-bit 0x104D8 */ - MMU_STAGE2__ST2MMU__AS3__FAULTSTATUS, /* (RO) 32-bit 0x104DC */ - MMU_STAGE2__ST2MMU__AS3__FAULTADDRESS, /* (RO) 64-bit 0x104E0 */ - MMU_STAGE2__ST2MMU__AS3__STATUS, /* (RO) 32-bit 0x104E8 */ - MMU_STAGE2__ST2MMU__AS3__TRANSCFG, /* (RW) 64-bit 0x104F0 */ - MMU_STAGE2__ST2MMU__AS3__FAULTEXTRA, /* (RO) 64-bit 0x104F8 */ - MMU_STAGE2__ST2MMU__AS4__TRANSTAB, /* (RW) 64-bit 0x10500 */ - MMU_STAGE2__ST2MMU__AS4__MEMATTR, /* (RW) 64-bit 0x10508 */ - MMU_STAGE2__ST2MMU__AS4__LOCKADDR, /* (RW) 64-bit 0x10510 */ - MMU_STAGE2__ST2MMU__AS4__COMMAND, /* (WO) 32-bit 0x10518 */ - MMU_STAGE2__ST2MMU__AS4__FAULTSTATUS, /* (RO) 32-bit 0x1051C */ - MMU_STAGE2__ST2MMU__AS4__FAULTADDRESS, /* (RO) 64-bit 0x10520 */ - MMU_STAGE2__ST2MMU__AS4__STATUS, /* (RO) 32-bit 0x10528 */ - MMU_STAGE2__ST2MMU__AS4__TRANSCFG, /* (RW) 64-bit 0x10530 */ - MMU_STAGE2__ST2MMU__AS4__FAULTEXTRA, /* (RO) 64-bit 0x10538 */ - MMU_STAGE2__ST2MMU__AS5__TRANSTAB, /* (RW) 64-bit 0x10540 */ - MMU_STAGE2__ST2MMU__AS5__MEMATTR, /* (RW) 64-bit 0x10548 */ - MMU_STAGE2__ST2MMU__AS5__LOCKADDR, /* (RW) 64-bit 0x10550 */ - MMU_STAGE2__ST2MMU__AS5__COMMAND, /* (WO) 32-bit 0x10558 */ - MMU_STAGE2__ST2MMU__AS5__FAULTSTATUS, /* (RO) 32-bit 0x1055C */ - MMU_STAGE2__ST2MMU__AS5__FAULTADDRESS, /* (RO) 64-bit 0x10560 */ - MMU_STAGE2__ST2MMU__AS5__STATUS, /* (RO) 32-bit 0x10568 */ - MMU_STAGE2__ST2MMU__AS5__TRANSCFG, /* (RW) 64-bit 0x10570 */ - MMU_STAGE2__ST2MMU__AS5__FAULTEXTRA, /* (RO) 64-bit 0x10578 */ - MMU_STAGE2__ST2MMU__AS6__TRANSTAB, /* (RW) 64-bit 0x10580 */ - MMU_STAGE2__ST2MMU__AS6__MEMATTR, /* (RW) 64-bit 0x10588 */ - MMU_STAGE2__ST2MMU__AS6__LOCKADDR, /* (RW) 64-bit 0x10590 */ - MMU_STAGE2__ST2MMU__AS6__COMMAND, /* (WO) 32-bit 0x10598 */ - MMU_STAGE2__ST2MMU__AS6__FAULTSTATUS, /* (RO) 32-bit 0x1059C */ - MMU_STAGE2__ST2MMU__AS6__FAULTADDRESS, /* (RO) 64-bit 0x105A0 */ - MMU_STAGE2__ST2MMU__AS6__STATUS, /* (RO) 32-bit 0x105A8 */ - MMU_STAGE2__ST2MMU__AS6__TRANSCFG, /* (RW) 64-bit 0x105B0 */ - MMU_STAGE2__ST2MMU__AS6__FAULTEXTRA, /* (RO) 64-bit 0x105B8 */ - MMU_STAGE2__ST2MMU__AS7__TRANSTAB, /* (RW) 64-bit 0x105C0 */ - MMU_STAGE2__ST2MMU__AS7__MEMATTR, /* (RW) 64-bit 0x105C8 */ - MMU_STAGE2__ST2MMU__AS7__LOCKADDR, /* (RW) 64-bit 0x105D0 */ - MMU_STAGE2__ST2MMU__AS7__COMMAND, /* (WO) 32-bit 0x105D8 */ - MMU_STAGE2__ST2MMU__AS7__FAULTSTATUS, /* (RO) 32-bit 0x105DC */ - MMU_STAGE2__ST2MMU__AS7__FAULTADDRESS, /* (RO) 64-bit 0x105E0 */ - MMU_STAGE2__ST2MMU__AS7__STATUS, /* (RO) 32-bit 0x105E8 */ - MMU_STAGE2__ST2MMU__AS7__TRANSCFG, /* (RW) 64-bit 0x105F0 */ - MMU_STAGE2__ST2MMU__AS7__FAULTEXTRA, /* (RO) 64-bit 0x105F8 */ - MMU_STAGE2__ST2MMU__AS8__TRANSTAB, /* (RW) 64-bit 0x10600 */ - MMU_STAGE2__ST2MMU__AS8__MEMATTR, /* (RW) 64-bit 0x10608 */ - MMU_STAGE2__ST2MMU__AS8__LOCKADDR, /* (RW) 64-bit 0x10610 */ - MMU_STAGE2__ST2MMU__AS8__COMMAND, /* (WO) 32-bit 0x10618 */ - MMU_STAGE2__ST2MMU__AS8__FAULTSTATUS, /* (RO) 32-bit 0x1061C */ - MMU_STAGE2__ST2MMU__AS8__FAULTADDRESS, /* (RO) 64-bit 0x10620 */ - MMU_STAGE2__ST2MMU__AS8__STATUS, /* (RO) 32-bit 0x10628 */ - MMU_STAGE2__ST2MMU__AS8__TRANSCFG, /* (RW) 64-bit 0x10630 */ - MMU_STAGE2__ST2MMU__AS8__FAULTEXTRA, /* (RO) 64-bit 0x10638 */ - MMU_STAGE2__ST2MMU__AS9__TRANSTAB, /* (RW) 64-bit 0x10640 */ - MMU_STAGE2__ST2MMU__AS9__MEMATTR, /* (RW) 64-bit 0x10648 */ - MMU_STAGE2__ST2MMU__AS9__LOCKADDR, /* (RW) 64-bit 0x10650 */ - MMU_STAGE2__ST2MMU__AS9__COMMAND, /* (WO) 32-bit 0x10658 */ - MMU_STAGE2__ST2MMU__AS9__FAULTSTATUS, /* (RO) 32-bit 0x1065C */ - MMU_STAGE2__ST2MMU__AS9__FAULTADDRESS, /* (RO) 64-bit 0x10660 */ - MMU_STAGE2__ST2MMU__AS9__STATUS, /* (RO) 32-bit 0x10668 */ - MMU_STAGE2__ST2MMU__AS9__TRANSCFG, /* (RW) 64-bit 0x10670 */ - MMU_STAGE2__ST2MMU__AS9__FAULTEXTRA, /* (RO) 64-bit 0x10678 */ - MMU_STAGE2__ST2MMU__AS10__TRANSTAB, /* (RW) 64-bit 0x10680 */ - MMU_STAGE2__ST2MMU__AS10__MEMATTR, /* (RW) 64-bit 0x10688 */ - MMU_STAGE2__ST2MMU__AS10__LOCKADDR, /* (RW) 64-bit 0x10690 */ - MMU_STAGE2__ST2MMU__AS10__COMMAND, /* (WO) 32-bit 0x10698 */ - MMU_STAGE2__ST2MMU__AS10__FAULTSTATUS, /* (RO) 32-bit 0x1069C */ - MMU_STAGE2__ST2MMU__AS10__FAULTADDRESS, /* (RO) 64-bit 0x106A0 */ - MMU_STAGE2__ST2MMU__AS10__STATUS, /* (RO) 32-bit 0x106A8 */ - MMU_STAGE2__ST2MMU__AS10__TRANSCFG, /* (RW) 64-bit 0x106B0 */ - MMU_STAGE2__ST2MMU__AS10__FAULTEXTRA, /* (RO) 64-bit 0x106B8 */ - MMU_STAGE2__ST2MMU__AS11__TRANSTAB, /* (RW) 64-bit 0x106C0 */ - MMU_STAGE2__ST2MMU__AS11__MEMATTR, /* (RW) 64-bit 0x106C8 */ - MMU_STAGE2__ST2MMU__AS11__LOCKADDR, /* (RW) 64-bit 0x106D0 */ - MMU_STAGE2__ST2MMU__AS11__COMMAND, /* (WO) 32-bit 0x106D8 */ - MMU_STAGE2__ST2MMU__AS11__FAULTSTATUS, /* (RO) 32-bit 0x106DC */ - MMU_STAGE2__ST2MMU__AS11__FAULTADDRESS, /* (RO) 64-bit 0x106E0 */ - MMU_STAGE2__ST2MMU__AS11__STATUS, /* (RO) 32-bit 0x106E8 */ - MMU_STAGE2__ST2MMU__AS11__TRANSCFG, /* (RW) 64-bit 0x106F0 */ - MMU_STAGE2__ST2MMU__AS11__FAULTEXTRA, /* (RO) 64-bit 0x106F8 */ - MMU_STAGE2__ST2MMU__AS12__TRANSTAB, /* (RW) 64-bit 0x10700 */ - MMU_STAGE2__ST2MMU__AS12__MEMATTR, /* (RW) 64-bit 0x10708 */ - MMU_STAGE2__ST2MMU__AS12__LOCKADDR, /* (RW) 64-bit 0x10710 */ - MMU_STAGE2__ST2MMU__AS12__COMMAND, /* (WO) 32-bit 0x10718 */ - MMU_STAGE2__ST2MMU__AS12__FAULTSTATUS, /* (RO) 32-bit 0x1071C */ - MMU_STAGE2__ST2MMU__AS12__FAULTADDRESS, /* (RO) 64-bit 0x10720 */ - MMU_STAGE2__ST2MMU__AS12__STATUS, /* (RO) 32-bit 0x10728 */ - MMU_STAGE2__ST2MMU__AS12__TRANSCFG, /* (RW) 64-bit 0x10730 */ - MMU_STAGE2__ST2MMU__AS12__FAULTEXTRA, /* (RO) 64-bit 0x10738 */ - MMU_STAGE2__ST2MMU__AS13__TRANSTAB, /* (RW) 64-bit 0x10740 */ - MMU_STAGE2__ST2MMU__AS13__MEMATTR, /* (RW) 64-bit 0x10748 */ - MMU_STAGE2__ST2MMU__AS13__LOCKADDR, /* (RW) 64-bit 0x10750 */ - MMU_STAGE2__ST2MMU__AS13__COMMAND, /* (WO) 32-bit 0x10758 */ - MMU_STAGE2__ST2MMU__AS13__FAULTSTATUS, /* (RO) 32-bit 0x1075C */ - MMU_STAGE2__ST2MMU__AS13__FAULTADDRESS, /* (RO) 64-bit 0x10760 */ - MMU_STAGE2__ST2MMU__AS13__STATUS, /* (RO) 32-bit 0x10768 */ - MMU_STAGE2__ST2MMU__AS13__TRANSCFG, /* (RW) 64-bit 0x10770 */ - MMU_STAGE2__ST2MMU__AS13__FAULTEXTRA, /* (RO) 64-bit 0x10778 */ - MMU_STAGE2__ST2MMU__AS14__TRANSTAB, /* (RW) 64-bit 0x10780 */ - MMU_STAGE2__ST2MMU__AS14__MEMATTR, /* (RW) 64-bit 0x10788 */ - MMU_STAGE2__ST2MMU__AS14__LOCKADDR, /* (RW) 64-bit 0x10790 */ - MMU_STAGE2__ST2MMU__AS14__COMMAND, /* (WO) 32-bit 0x10798 */ - MMU_STAGE2__ST2MMU__AS14__FAULTSTATUS, /* (RO) 32-bit 0x1079C */ - MMU_STAGE2__ST2MMU__AS14__FAULTADDRESS, /* (RO) 64-bit 0x107A0 */ - MMU_STAGE2__ST2MMU__AS14__STATUS, /* (RO) 32-bit 0x107A8 */ - MMU_STAGE2__ST2MMU__AS14__TRANSCFG, /* (RW) 64-bit 0x107B0 */ - MMU_STAGE2__ST2MMU__AS14__FAULTEXTRA, /* (RO) 64-bit 0x107B8 */ - MMU_STAGE2__ST2MMU__AS15__TRANSTAB, /* (RW) 64-bit 0x107C0 */ - MMU_STAGE2__ST2MMU__AS15__MEMATTR, /* (RW) 64-bit 0x107C8 */ - MMU_STAGE2__ST2MMU__AS15__LOCKADDR, /* (RW) 64-bit 0x107D0 */ - MMU_STAGE2__ST2MMU__AS15__COMMAND, /* (WO) 32-bit 0x107D8 */ - MMU_STAGE2__ST2MMU__AS15__FAULTSTATUS, /* (RO) 32-bit 0x107DC */ - MMU_STAGE2__ST2MMU__AS15__FAULTADDRESS, /* (RO) 64-bit 0x107E0 */ - MMU_STAGE2__ST2MMU__AS15__STATUS, /* (RO) 32-bit 0x107E8 */ - MMU_STAGE2__ST2MMU__AS15__TRANSCFG, /* (RW) 64-bit 0x107F0 */ - MMU_STAGE2__ST2MMU__AS15__FAULTEXTRA, /* (RO) 64-bit 0x107F8 */ - NR_V6_0_REGS, -}; - -enum kbase_regmap_enum_v6_2 { - GPU_CONTROL__REVIDR = NR_V6_0_REGS, /* (RO) 32-bit 0x280 */ - GPU_CONTROL__STACK_PRESENT, /* (RO) 64-bit 0xE00 */ - GPU_CONTROL__STACK_PWROFF, /* (WO) 64-bit 0xE30 */ - GPU_CONTROL__STACK_PWRON, /* (WO) 64-bit 0xE20 */ - GPU_CONTROL__STACK_PWRTRANS, /* (RO) 64-bit 0xE40 */ - GPU_CONTROL__STACK_READY, /* (RO) 64-bit 0xE10 */ - NR_V6_2_REGS, -}; - -enum kbase_regmap_enum_v7_0 { - GPU_CONTROL__TEXTURE_FEATURES_3 = NR_V6_2_REGS, /* (RO) 32-bit 0xBC */ - NR_V7_0_REGS, -}; - -enum kbase_regmap_enum_v7_2 { - GPU_CONTROL__CORE_FEATURES = NR_V7_0_REGS, /* (RO) 32-bit 0x8 */ - GPU_CONTROL__THREAD_TLS_ALLOC, /* (RO) 32-bit 0x310 */ - NR_V7_2_REGS, -}; - -enum kbase_regmap_enum_v9_0 { - NR_V9_0_REGS = NR_V7_2_REGS, -}; - -/* - * V9_0_REMOVED_REGS: - * GPU_CONTROL__CORE_FEATURES - * GPU_CONTROL__THREAD_TLS_ALLOC - * JOB_CONTROL__JOB_IRQ_THROTTLE - */ - -enum kbase_regmap_enum_v9_2 { - GPU_CONTROL__L2_CONFIG = NR_V9_0_REGS, /* (RW) 32-bit 0x48 */ - NR_V9_2_REGS, -}; - -enum kbase_regmap_enum_v9_14 { - PTM_AW_IRQ_CLEAR = NR_V9_2_REGS, /* (RW) 32-bit 0x1FFC8 */ - PTM_AW_IRQ_INJECTION, /* (RW) 32-bit 0x1FFD4 */ - PTM_AW_IRQ_MASK, /* (RW) 32-bit 0x1FFCC */ - PTM_AW_IRQ_RAWSTAT, /* (RO) 32-bit 0x1FFC4 */ - PTM_AW_IRQ_STATUS, /* (RO) 32-bit 0x1FFD0 */ - PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE0, /* (RO) 32-bit 0x1FFD8 */ - PTM_AW_MESSAGE__PTM_INCOMING_MESSAGE1, /* (RO) 32-bit 0x1FFDC */ - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE0, /* (RW) 32-bit 0x1FFE4 */ - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1, /* (RW) 32-bit 0x1FFE8 */ - PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS, /* (RO) 32-bit 0x1FFE0 */ - PTM_ID, /* (RO) 32-bit 0x1FFC0 */ - NR_V9_14_REGS, -}; - -#endif /* _MALI_KBASE_REGMAP_JM_ENUMS_H_ */ diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_macros.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_macros.h deleted file mode 100644 index 27d2702a06ce..000000000000 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_jm_macros.h +++ /dev/null @@ -1,308 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _MALI_KBASE_REGMAP_JM_MACROS_H_ -#define _MALI_KBASE_REGMAP_JM_MACROS_H_ - -#if MALI_USE_CSF -#error "Cannot be compiled with CSF" -#endif - -#define ENUM_OFFSET(_index, _base, _next) (_base + _index * (_next - _base)) - -#define GPU_CONTROL_ENUM(regname) GPU_CONTROL__##regname -#define GPU_TEXTURE_FEATURES_ENUM(n) GPU_CONTROL_ENUM(TEXTURE_FEATURES_##n) -#define GPU_TEXTURE_FEATURES_OFFSET(n) (GPU_TEXTURE_FEATURES_ENUM(0) + n) -#define GPU_JS_FEATURES_ENUM(n) GPU_CONTROL_ENUM(JS##n##_FEATURES) -#define GPU_JS_FEATURES_OFFSET(n) (GPU_JS_FEATURES_ENUM(0) + n) - -#define JOB_CONTROL_ENUM(regname) JOB_CONTROL__##regname -#define JOB_SLOT_ENUM(n, regname) JOB_CONTROL_ENUM(JS##n##__##regname) -#define JOB_SLOT_BASE_ENUM(n) JOB_SLOT_ENUM(n, HEAD) -#define JOB_SLOT_OFFSET(n, regname) \ - ENUM_OFFSET(n, JOB_SLOT_ENUM(0, regname), JOB_SLOT_ENUM(1, regname)) -#define JOB_SLOT_BASE_OFFSET(n) JOB_SLOT_OFFSET(n, HEAD) - -#define MMU_CONTROL_ENUM(regname) MMU_STAGE1__ST1MMU__##regname -#define MMU_AS_ENUM(n, regname) MMU_CONTROL_ENUM(AS##n##__##regname) -#define MMU_AS_BASE_ENUM(n) MMU_AS_ENUM(n, TRANSTAB) -#define MMU_AS_OFFSET(n, regname) ENUM_OFFSET(n, MMU_AS_ENUM(0, regname), MMU_AS_ENUM(1, regname)) -#define MMU_AS_BASE_OFFSET(n) MMU_AS_OFFSET(n, TRANSTAB) - -#define PTM_AW_ENUM(regname) PTM_AW__##regname -#define PTM_AW_MESSAGE_ENUM(regname) PTM_AW_MESSAGE__##regname - -/* register value macros */ -/* GPU_STATUS values */ -#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ -#define GPU_STATUS_CYCLE_COUNT_ACTIVE (1 << 6) /* Set if the cycle counter is active. */ - -/* PRFCNT_CONFIG register values */ -#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ -#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ -#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ - -/* The performance counters are disabled. */ -#define PRFCNT_CONFIG_MODE_OFF 0 -/* The performance counters are enabled, but are only written out when a - * PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. - */ -#define PRFCNT_CONFIG_MODE_MANUAL 1 -/* The performance counters are enabled, and are written out each time a tile - * finishes rendering. - */ -#define PRFCNT_CONFIG_MODE_TILE 2 - -/* - * Begin AARCH64 MMU TRANSTAB register values - */ -#define AS_TRANSTAB_BASE_SHIFT GPU_U(4) -#define AS_TRANSTAB_BASE_MASK (GPU_ULL(0xFFFFFFFFFFFFFFF) << AS_TRANSTAB_BASE_SHIFT) -#define AS_TRANSTAB_BASE_GET(reg_val) (((reg_val)&AS_TRANSTAB_BASE_MASK) >> AS_TRANSTAB_BASE_SHIFT) -#define AS_TRANSTAB_BASE_SET(reg_val, value) \ - (~(~(reg_val) | AS_TRANSTAB_BASE_MASK) | \ - (((uint64_t)(value) << AS_TRANSTAB_BASE_SHIFT) & AS_TRANSTAB_BASE_MASK)) - -#define AS_FAULTSTATUS_EXCEPTION_TYPE_OK 0x0 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_DONE 0x1 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_STOPPED 0x3 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TERMINATED 0x4 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_KABOOM 0x5 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_EUREKA 0x6 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ACTIVE 0x8 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_CONFIG_FAULT 0x40 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_POWER_FAULT 0x41 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_READ_FAULT 0x42 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_WRITE_FAULT 0x43 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_AFFINITY_FAULT 0x44 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_JOB_BUS_FAULT 0x48 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_INSTR_INVALID_PC 0x50 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_INSTR_INVALID_ENC 0x51 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_INSTR_BARRIER_FAULT 0x55 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_DATA_INVALID_FAULT 0x58 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TILE_RANGE_FAULT 0x59 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDR_RANGE_FAULT 0x5A -#define AS_FAULTSTATUS_EXCEPTION_TYPE_IMPRECISE_FAULT 0x5B -#define AS_FAULTSTATUS_EXCEPTION_TYPE_OUT_OF_MEMORY 0x60 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_UNKNOWN 0x7F -#define AS_FAULTSTATUS_EXCEPTION_TYPE_DELAYED_BUS_FAULT 0x80 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_GPU_SHAREABILITY_FAULT 0x88 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_SYSTEM_SHAREABILITY_FAULT 0x89 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_GPU_CACHEABILITY_FAULT 0x8A -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_0 0xC0 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_1 0xC1 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_2 0xC2 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_3 0xC3 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_4 0xC4 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_IDENTITY 0xC7 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_PERMISSION_FAULT_0 0xC8 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_PERMISSION_FAULT_1 0xC9 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_PERMISSION_FAULT_2 0xCA -#define AS_FAULTSTATUS_EXCEPTION_TYPE_PERMISSION_FAULT_3 0xCB -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_0 0xD0 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_1 0xD1 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_2 0xD2 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_3 0xD3 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_0 0xD8 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_1 0xD9 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_2 0xDA -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_3 0xDB -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN0 0xE0 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN1 0xE1 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN2 0xE2 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN3 0xE3 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT0 0xE4 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT1 0xE5 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT2 0xE6 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT3 0xE7 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_0 0xE8 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_1 0xE9 -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_2 0xEA -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_3 0xEB -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_0 0xEC -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_1 0xED -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_2 0xEE -#define AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_3 0xEF - -/* No JM-specific MMU control registers */ -/* No JM-specific MMU address space control registers */ - -/* JS_COMMAND register commands */ -#define JS_COMMAND_NOP 0x00 /* NOP Operation. Writing this value is ignored */ -#define JS_COMMAND_START 0x01 /* Start processing a job chain. Writing this value is ignored */ -#define JS_COMMAND_SOFT_STOP 0x02 /* Gently stop processing a job chain */ -#define JS_COMMAND_HARD_STOP 0x03 /* Rudely stop processing a job chain */ -#define JS_COMMAND_SOFT_STOP_0 0x04 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */ -#define JS_COMMAND_HARD_STOP_0 0x05 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */ -#define JS_COMMAND_SOFT_STOP_1 0x06 /* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */ -#define JS_COMMAND_HARD_STOP_1 0x07 /* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */ - -#define JS_COMMAND_MASK 0x07 /* Mask of bits currently in use by the HW */ - -/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */ -#define JS_CONFIG_START_FLUSH_NO_ACTION (0u << 0) -#define JS_CONFIG_START_FLUSH_CLEAN (1u << 8) -#define JS_CONFIG_START_FLUSH_INV_SHADER_OTHER (2u << 8) -#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE (3u << 8) -#define JS_CONFIG_START_MMU (1u << 10) -#define JS_CONFIG_JOB_CHAIN_FLAG (1u << 11) -#define JS_CONFIG_END_FLUSH_NO_ACTION JS_CONFIG_START_FLUSH_NO_ACTION -#define JS_CONFIG_END_FLUSH_CLEAN (1u << 12) -#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE (3u << 12) -#define JS_CONFIG_ENABLE_FLUSH_REDUCTION (1u << 14) -#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK (1u << 15) -#define JS_CONFIG_THREAD_PRI(n) ((n) << 16) - -/* JS_XAFFINITY register values */ -#define JS_XAFFINITY_XAFFINITY_ENABLE (1u << 0) -#define JS_XAFFINITY_TILER_ENABLE (1u << 8) -#define JS_XAFFINITY_CACHE_ENABLE (1u << 16) - -/* JS_STATUS register values */ - -/* NOTE: Please keep this values in sync with enum base_jd_event_code in mali_base_kernel.h. - * The values are separated to avoid dependency of userspace and kernel code. - */ - -/* Group of values representing the job status instead of a particular fault */ -#define JS_STATUS_NO_EXCEPTION_BASE 0x00 -#define JS_STATUS_INTERRUPTED (JS_STATUS_NO_EXCEPTION_BASE + 0x02) /* 0x02 means INTERRUPTED */ -#define JS_STATUS_STOPPED (JS_STATUS_NO_EXCEPTION_BASE + 0x03) /* 0x03 means STOPPED */ -#define JS_STATUS_TERMINATED (JS_STATUS_NO_EXCEPTION_BASE + 0x04) /* 0x04 means TERMINATED */ - -/* General fault values */ -#define JS_STATUS_FAULT_BASE 0x40 -#define JS_STATUS_CONFIG_FAULT (JS_STATUS_FAULT_BASE) /* 0x40 means CONFIG FAULT */ -#define JS_STATUS_POWER_FAULT (JS_STATUS_FAULT_BASE + 0x01) /* 0x41 means POWER FAULT */ -#define JS_STATUS_READ_FAULT (JS_STATUS_FAULT_BASE + 0x02) /* 0x42 means READ FAULT */ -#define JS_STATUS_WRITE_FAULT (JS_STATUS_FAULT_BASE + 0x03) /* 0x43 means WRITE FAULT */ -#define JS_STATUS_AFFINITY_FAULT (JS_STATUS_FAULT_BASE + 0x04) /* 0x44 means AFFINITY FAULT */ -#define JS_STATUS_BUS_FAULT (JS_STATUS_FAULT_BASE + 0x08) /* 0x48 means BUS FAULT */ - -/* Instruction or data faults */ -#define JS_STATUS_INSTRUCTION_FAULT_BASE 0x50 -#define JS_STATUS_INSTR_INVALID_PC \ - (JS_STATUS_INSTRUCTION_FAULT_BASE) /* 0x50 means INSTR INVALID PC */ -#define JS_STATUS_INSTR_INVALID_ENC \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x01) /* 0x51 means INSTR INVALID ENC */ -#define JS_STATUS_INSTR_TYPE_MISMATCH \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x02) /* 0x52 means INSTR TYPE MISMATCH */ -#define JS_STATUS_INSTR_OPERAND_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x03) /* 0x53 means INSTR OPERAND FAULT */ -#define JS_STATUS_INSTR_TLS_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x04) /* 0x54 means INSTR TLS FAULT */ -#define JS_STATUS_INSTR_BARRIER_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x05) /* 0x55 means INSTR BARRIER FAULT */ -#define JS_STATUS_INSTR_ALIGN_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x06) /* 0x56 means INSTR ALIGN FAULT */ -/* NOTE: No fault with 0x57 code defined in spec. */ -#define JS_STATUS_DATA_INVALID_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x08) /* 0x58 means DATA INVALID FAULT */ -#define JS_STATUS_TILE_RANGE_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x09) /* 0x59 means TILE RANGE FAULT */ -#define JS_STATUS_ADDRESS_RANGE_FAULT \ - (JS_STATUS_INSTRUCTION_FAULT_BASE + 0x0A) /* 0x5A means ADDRESS RANGE FAULT */ - -/* Other faults */ -#define JS_STATUS_MEMORY_FAULT_BASE 0x60 -#define JS_STATUS_OUT_OF_MEMORY (JS_STATUS_MEMORY_FAULT_BASE) /* 0x60 means OUT OF MEMORY */ -#define JS_STATUS_UNKNOWN 0x7F /* 0x7F means UNKNOWN */ - -/* JS_FEATURES register */ -#define JS_FEATURE_NULL_JOB (1u << 1) -#define JS_FEATURE_SET_VALUE_JOB (1u << 2) -#define JS_FEATURE_CACHE_FLUSH_JOB (1u << 3) -#define JS_FEATURE_COMPUTE_JOB (1u << 4) -#define JS_FEATURE_VERTEX_JOB (1u << 5) -#define JS_FEATURE_GEOMETRY_JOB (1u << 6) -#define JS_FEATURE_TILER_JOB (1u << 7) -#define JS_FEATURE_FUSED_JOB (1u << 8) -#define JS_FEATURE_FRAGMENT_JOB (1u << 9) - -/* JM_CONFIG register */ -#define JM_TIMESTAMP_OVERRIDE (1ul << 0) -#define JM_CLOCK_GATE_OVERRIDE (1ul << 1) -#define JM_JOB_THROTTLE_ENABLE (1ul << 2) -#define JM_JOB_THROTTLE_LIMIT_SHIFT (3) -#define JM_MAX_JOB_THROTTLE_LIMIT (0x3F) -#define JM_FORCE_COHERENCY_FEATURES_SHIFT (2) - -/* GPU_COMMAND values */ -#define GPU_COMMAND_NOP 0x00 /* No operation, nothing happens */ -#define GPU_COMMAND_SOFT_RESET \ - 0x01 /* Stop all external bus interfaces, and then reset the entire GPU. */ -#define GPU_COMMAND_HARD_RESET 0x02 /* Immediately reset the entire GPU. */ -#define GPU_COMMAND_PRFCNT_CLEAR \ - 0x03 /* Clear all performance counters, setting them all to zero. */ -#define GPU_COMMAND_PRFCNT_SAMPLE \ - 0x04 /* Sample all performance counters, writing them out to memory */ -#define GPU_COMMAND_CYCLE_COUNT_START \ - 0x05 /* Starts the cycle counter, and system timestamp propagation */ -#define GPU_COMMAND_CYCLE_COUNT_STOP \ - 0x06 /* Stops the cycle counter, and system timestamp propagation */ -#define GPU_COMMAND_CLEAN_CACHES 0x07 /* Clean all caches */ -#define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */ -#define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */ - -/* GPU_COMMAND cache flush alias to CSF command payload */ -#define GPU_COMMAND_CACHE_CLN_INV_L2 GPU_COMMAND_CLEAN_INV_CACHES -#define GPU_COMMAND_CACHE_CLN_INV_L2_LSC GPU_COMMAND_CLEAN_INV_CACHES -#define GPU_COMMAND_CACHE_CLN_INV_FULL GPU_COMMAND_CLEAN_INV_CACHES -#define GPU_COMMAND_CACHE_CLN_INV_LSC GPU_COMMAND_CLEAN_INV_CACHES - -/* Merge cache flush commands */ -#define GPU_COMMAND_FLUSH_CACHE_MERGE(cmd1, cmd2) ((cmd1) > (cmd2) ? (cmd1) : (cmd2)) - -/* IRQ flags */ -#define GPU_FAULT (1U << 0) /* A GPU Fault has occurred */ -#define MULTIPLE_GPU_FAULTS (1U << 7) /* More than one GPU Fault occurred. */ -#define RESET_COMPLETED (1U << 8) /* Set when a reset has completed. */ -#define POWER_CHANGED_SINGLE \ - (1U << 9) /* Set when a single core has finished powering up or down. */ -#define POWER_CHANGED_ALL (1U << 10) /* Set when all cores have finished powering up or down. */ -#define PRFCNT_SAMPLE_COMPLETED (1U << 16) /* Set when a performance count sample has completed. */ -#define CLEAN_CACHES_COMPLETED (1U << 17) /* Set when a cache clean operation has completed. */ -#define FLUSH_PA_RANGE_COMPLETED \ - (1 << 20) /* Set when a physical range cache clean operation has completed. */ - -/* - * In Debug build, - * GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE is used to clear and enable - * interrupts sources of GPU_IRQ by writing it onto GPU_IRQ_CLEAR/MASK registers. - * - * In Release build, - * GPU_IRQ_REG_COMMON is used. - * - * Note: - * CLEAN_CACHES_COMPLETED - Used separately for cache operation. - */ -#define GPU_IRQ_REG_COMMON \ - (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED | POWER_CHANGED_ALL | \ - PRFCNT_SAMPLE_COMPLETED) - -#define WINDOW_IRQ_MESSAGE (1U << 0) -#define WINDOW_IRQ_INVALID_ACCESS (1U << 1) -#define WINDOW_IRQ_GPU (1U << 2) -#define WINDOW_IRQ_JOB (1U << 3) -#define WINDOW_IRQ_MMU (1U << 4) -#define WINDOW_IRQ_EVENT (1U << 5) - -#endif /* _MALI_KBASE_REGMAP_JM_MACROS_H_ */ diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_csf.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_csf.h index 0e3a56e9d664..0678cf76fc0a 100644 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_csf.h +++ b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_csf.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,9 +22,6 @@ #ifndef _MALI_KBASE_REGMAP_LEGACY_CSF_H_ #define _MALI_KBASE_REGMAP_LEGACY_CSF_H_ -#if !MALI_USE_CSF && defined(__KERNEL__) -#error "Cannot be compiled with JM" -#endif /* GPU control registers */ #define MCU_CONTROL 0x700 diff --git a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_jm.h b/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_jm.h deleted file mode 100644 index c1a54991f4c6..000000000000 --- a/drivers/gpu/arm/valhall/hw_access/regmap/mali_kbase_regmap_legacy_jm.h +++ /dev/null @@ -1,108 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _MALI_KBASE_REGMAP_LEGACY_JM_H_ -#define _MALI_KBASE_REGMAP_LEGACY_JM_H_ - -#if MALI_USE_CSF && defined(__KERNEL__) -#error "Cannot be compiled with CSF" -#endif - -/* GPU control registers */ -#define CORE_FEATURES 0x008 /* (RO) Shader Core Features */ -#define JS_PRESENT 0x01C /* (RO) Job slots present */ -#define LATEST_FLUSH 0x038 /* (RO) Flush ID of latest clean-and-invalidate operation */ -#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory region base address, low word */ -#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory region base address, high word */ -#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter configuration */ -#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable flags for Job Manager */ -#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable flags for shader cores */ -#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable flags for tiler */ -#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable flags for MMU/L2 cache */ - -#define JS0_FEATURES 0x0C0 /* (RO) Features of job slot 0 */ -#define JS1_FEATURES 0x0C4 /* (RO) Features of job slot 1 */ -#define JS2_FEATURES 0x0C8 /* (RO) Features of job slot 2 */ -#define JS3_FEATURES 0x0CC /* (RO) Features of job slot 3 */ -#define JS4_FEATURES 0x0D0 /* (RO) Features of job slot 4 */ -#define JS5_FEATURES 0x0D4 /* (RO) Features of job slot 5 */ -#define JS6_FEATURES 0x0D8 /* (RO) Features of job slot 6 */ -#define JS7_FEATURES 0x0DC /* (RO) Features of job slot 7 */ -#define JS8_FEATURES 0x0E0 /* (RO) Features of job slot 8 */ -#define JS9_FEATURES 0x0E4 /* (RO) Features of job slot 9 */ -#define JS10_FEATURES 0x0E8 /* (RO) Features of job slot 10 */ -#define JS11_FEATURES 0x0EC /* (RO) Features of job slot 11 */ -#define JS12_FEATURES 0x0F0 /* (RO) Features of job slot 12 */ -#define JS13_FEATURES 0x0F4 /* (RO) Features of job slot 13 */ -#define JS14_FEATURES 0x0F8 /* (RO) Features of job slot 14 */ -#define JS15_FEATURES 0x0FC /* (RO) Features of job slot 15 */ - -#define JS_FEATURES_REG(n) GPU_CONTROL_REG(JS0_FEATURES + ((n) << 2)) - -#define JM_CONFIG 0xF00 /* (RW) Job manager configuration (implementation-specific) */ - -/* Job control registers */ -/* status==active and _next == busy snapshot from last JOB_IRQ_CLEAR */ -#define JOB_IRQ_JS_STATE 0x010 -/* cycles to delay delivering an interrupt externally. The JOB_IRQ_STATUS - * is NOT affected by this, just the delivery of the interrupt. - */ -#define JOB_IRQ_THROTTLE 0x014 - -#define JOB_SLOT0 0x800 /* Configuration registers for job slot 0 */ -#define JOB_SLOT_REG(n, r) (JOB_CONTROL_REG(JOB_SLOT0 + ((n) << 7)) + (r)) -#define JOB_SLOT1 0x880 /* Configuration registers for job slot 1 */ -#define JOB_SLOT2 0x900 /* Configuration registers for job slot 2 */ -#define JOB_SLOT3 0x980 /* Configuration registers for job slot 3 */ -#define JOB_SLOT4 0xA00 /* Configuration registers for job slot 4 */ -#define JOB_SLOT5 0xA80 /* Configuration registers for job slot 5 */ -#define JOB_SLOT6 0xB00 /* Configuration registers for job slot 6 */ -#define JOB_SLOT7 0xB80 /* Configuration registers for job slot 7 */ -#define JOB_SLOT8 0xC00 /* Configuration registers for job slot 8 */ -#define JOB_SLOT9 0xC80 /* Configuration registers for job slot 9 */ -#define JOB_SLOT10 0xD00 /* Configuration registers for job slot 10 */ -#define JOB_SLOT11 0xD80 /* Configuration registers for job slot 11 */ -#define JOB_SLOT12 0xE00 /* Configuration registers for job slot 12 */ -#define JOB_SLOT13 0xE80 /* Configuration registers for job slot 13 */ -#define JOB_SLOT14 0xF00 /* Configuration registers for job slot 14 */ -#define JOB_SLOT15 0xF80 /* Configuration registers for job slot 15 */ - -/* JM Job control register definitions for mali_kbase_debug_job_fault */ -#define JS_HEAD_LO 0x00 /* (RO) Job queue head pointer for job slot n, low word */ -#define JS_HEAD_HI 0x04 /* (RO) Job queue head pointer for job slot n, high word */ -#define JS_TAIL_LO 0x08 /* (RO) Job queue tail pointer for job slot n, low word */ -#define JS_TAIL_HI 0x0C /* (RO) Job queue tail pointer for job slot n, high word */ -#define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ -#define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ -#define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ -#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job slot n*/ -#define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ -#define JS_STATUS 0x24 /* (RO) Status register for job slot n */ -#define JS_HEAD_NEXT_LO 0x40 /* (RW) Next job queue head pointer for job slot n, low word */ -#define JS_HEAD_NEXT_HI 0x44 /* (RW) Next job queue head pointer for job slot n, high word */ -#define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ -#define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ -#define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ -#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for job slot n */ -#define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ -#define JS_FLUSH_ID_NEXT 0x70 /* (RW) Next job slot n cache flush ID */ - -#endif /* _MALI_KBASE_REGMAP_LEGACY_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/hwcnt/Kbuild b/drivers/gpu/arm/valhall/hwcnt/Kbuild index f26717e92e0f..2bd5848984f3 100644 --- a/drivers/gpu/arm/valhall/hwcnt/Kbuild +++ b/drivers/gpu/arm/valhall/hwcnt/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2022 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2022, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -23,14 +23,6 @@ valhall_kbase-y += \ hwcnt/mali_kbase_hwcnt_gpu.o \ hwcnt/mali_kbase_hwcnt_types.o \ hwcnt/mali_kbase_hwcnt_virtualizer.o \ - hwcnt/mali_kbase_hwcnt_watchdog_if_timer.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += \ - hwcnt/backend/mali_kbase_hwcnt_backend_csf.o \ - hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.o -else - valhall_kbase-y += \ - hwcnt/backend/mali_kbase_hwcnt_backend_jm.o \ - hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.o -endif + hwcnt/mali_kbase_hwcnt_watchdog_if_timer.o \ + hwcnt/backend/mali_kbase_hwcnt_backend_csf.o \ + hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.o diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c index 723ae207070e..7f33aa6e3942 100644 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c +++ b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c @@ -39,6 +39,9 @@ /* Used to check for a sample in which all counters in the block are disabled */ #define HWCNT_BLOCK_EMPTY_SAMPLE (2) +#define RING_BUF_REFCNT_MAX 2 +#define RING_BUF_REFCNT_MIN 0 + /** * enum kbase_hwcnt_backend_csf_dump_state - HWC CSF backend dumping states. * @@ -99,17 +102,26 @@ enum kbase_hwcnt_backend_csf_dump_state { * @KBASE_HWCNT_BACKEND_CSF_DISABLED: Initial state, and the state when backend * is disabled. * - * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: Enable request is in - * progress, waiting for firmware acknowledgment. + * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ: Backend is + * being enabled, pending to send enable REQ to firmware. + * + * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK: Enable REQ + * has been sent to FW, waiting for firmware acknowledgment. * * @KBASE_HWCNT_BACKEND_CSF_ENABLED: Enable request has been acknowledged, * enable is done. * - * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: Disable request is in - * progress, waiting for firmware acknowledgment. + * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ: Backend is + * being disabled, pending to send disable REQ to firmware. + * + * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK: Disable REQ + * has been sent to FW, waiting for firmware acknowledgment. * * @KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: Disable request has been * acknowledged, waiting for dump workers to be finished. + * Note: This is mainly used when recovering from a recoverable error. During a + * clean disable procedure, backend will transition directly to DISABLED state + * after being ACKed. * * @KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: An * unrecoverable error happened, waiting for dump workers to be finished. @@ -118,26 +130,52 @@ enum kbase_hwcnt_backend_csf_dump_state { * happened, and dump workers have finished, waiting for reset. * * Valid state transitions: - * DISABLED -> TRANSITIONING_TO_ENABLED (on enable) - * TRANSITIONING_TO_ENABLED -> ENABLED (on enable ack) - * ENABLED -> TRANSITIONING_TO_DISABLED (on disable) - * TRANSITIONING_TO_DISABLED -> DISABLED_WAIT_FOR_WORKER (on disable ack) - * DISABLED_WAIT_FOR_WORKER -> DISABLED (after workers are flushed) - * DISABLED -> UNRECOVERABLE_ERROR (on unrecoverable error) - * ANY but DISABLED -> UNRECOVERABLE_ERROR_WAIT_FOR_WORKER (on unrecoverable - * error) - * UNRECOVERABLE_ERROR -> DISABLED (on before reset) + * - DISABLED -> TRANSITIONING_TO_ENABLED_PENDING_REQ (on enable before sending REQ) + * - TRANSITIONING_TO_ENABLED_PENDING_REQ -> + * TRANSITIONING_TO_ENABLED_PENDING_ACK (on sending enable REQ) + * - TRANSITIONING_TO_ENABLED_PENDING_ACK -> ENABLED (on enable ack) + * - ENABLED -> TRANSITIONING_TO_DISABLED_PENDING_REQ (on disable before sending REQ) + * - TRANSITIONING_TO_DISABLED_PENDING_REQ -> + * TRANSITIONING_TO_DISABLED_PENDING_ACK (on sending disable REQ) + * - TRANSITIONING_TO_DISABLED_PENDING_ACK -> DISABLED (on disable ack) + * - TRANSITIONING_TO_DISABLED_PENDING_ACK -> DISABLED_WAIT_FOR_WORKER (on disable ack + * after a recoverable error) + * - DISABLED_WAIT_FOR_WORKER -> DISABLED (after workers are flushed) + * - UNRECOVERABLE_ERROR_WAIT_FOR_WORKER -> UNRECOVERABLE_ERROR (after workers are flushed) + * - DISABLED -> UNRECOVERABLE_ERROR (on unrecoverable error) + * - ANY but DISABLED -> UNRECOVERABLE_ERROR_WAIT_FOR_WORKER (on unrecoverable error) + * - UNRECOVERABLE_ERROR -> DISABLED (on before reset) + * + * More valid state transitions due to MCU being powered off: + * - TRANSITIONING_TO_ENABLED_PENDING_* -> + * TRANSITIONING_TO_DISABLED_PENDING_* (backend needs to be disabled + * but enable REQ hasn't been sent or ACK not received + * due to MCU being OFF) + * - TRANSITIONING_TO_DISABLED_PENDING_* -> + * TRANSITIONING_TO_ENABLED_PENDING_* (opposite case to the previous one) */ enum kbase_hwcnt_backend_csf_enable_state { KBASE_HWCNT_BACKEND_CSF_DISABLED, - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK, KBASE_HWCNT_BACKEND_CSF_ENABLED, - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK, KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER, KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER, KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR, }; +enum kbase_hwcnt_backend_block_type { + KBASE_HWCNT_BLOCK_METADATA, + KBASE_HWCNT_BLOCK_FW, + KBASE_HWCNT_BLOCK_CSHW, + KBASE_HWCNT_BLOCK_TILER, + KBASE_HWCNT_BLOCK_MEMSYS, + KBASE_HWCNT_BLOCK_SHADER, + KBASE_HWCNT_BLOCK_NEURAL +}; + /** * struct kbase_hwcnt_backend_csf_info - Information used to create an instance * of a CSF hardware counter backend. @@ -153,6 +191,19 @@ enum kbase_hwcnt_backend_csf_enable_state { * @prfcnt_info: Performance counter information. * @watchdog_if: Watchdog interface object pointer. * @watchdog_timer_interval_ms: Watchdog timer interval + * @last_transition: Last backend state transition obtained + * from the metadata block. + * @mcu_on: True if MCU is powered on. + * @ring_buf: Opaque pointer for ring buffer object. + * @ring_buf_cpu_base: CPU base address of the allocated ring buffer. + * @ring_buf_refcnt: Reference counter for ring buffer, used to control + * its allocation across MCU power cycles. + * @hwc_deferred_op_workq: Workqueue for deferred operations on the backend, + * e.g. due to MCU power cycle. + * @hwc_dump_reenable_work: Worker for re-enabling the backend after MCU ON notification. + * @hwc_ring_buf_free_work: Worker for freeing the ring buffer. + * @ring_buf_cleanup: Pointer to the ring buffer that needs to be freed. + * @pending_disable_req: True if disable REQ is pending to be sent when MCU is powered on. */ struct kbase_hwcnt_backend_csf_info { struct kbase_hwcnt_backend_csf *backend; @@ -165,6 +216,19 @@ struct kbase_hwcnt_backend_csf_info { struct kbase_hwcnt_backend_csf_if_prfcnt_info prfcnt_info; struct kbase_hwcnt_watchdog_interface *watchdog_if; u32 watchdog_timer_interval_ms; + enum kbase_hwcnt_backend_sample_reason last_transition; + bool mcu_on; + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf; + void *ring_buf_cpu_base; + struct kbasep_hwcnt_ring_buf_refcnt { + int val; + bool incremented_on_mcu_off; + } ring_buf_refcnt; + struct workqueue_struct *hwc_deferred_op_workq; + struct work_struct hwc_dump_reenable_work; + struct work_struct hwc_ring_buf_free_work; + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf_cleanup; + bool pending_disable_req; }; /** @@ -179,7 +243,10 @@ struct kbase_hwcnt_backend_csf_info { * @hw_block_cnt: Total number of hardware counter blocks. The hw counters blocks are * sub-categorized into 4 classes: front-end, tiler, memory system, and shader. * hw_block_cnt = fe_cnt + tiler_cnt + mmu_l2_cnt + shader_cnt. - * @block_cnt: Total block count (sum of all counter blocks: hw_block_cnt + fw_block_cnt). + * @md_block_cnt: Total number of metadata counter blocks. + * @ringbuf_block_cnt: Total block count (sum of all counter blocks: hw_block_cnt + fw_block_cnt + * + md_block_cnt). + * @payload_block_cnt: Total number of payload counter blocks: hw_block_cnt + fw_block_cnt. * @shader_avail_mask: Bitmap of all shader cores in the system. * @enable_mask_offset: Offset in array elements of enable mask in each block * starting from the beginning of block. @@ -195,7 +262,9 @@ struct kbase_hwcnt_csf_physical_layout { u8 shader_cnt; u8 fw_block_cnt; u8 hw_block_cnt; - u8 block_cnt; + u8 md_block_cnt; + u8 ringbuf_block_cnt; + u8 payload_block_cnt; u64 shader_avail_mask; size_t enable_mask_offset; size_t headers_per_block; @@ -229,8 +298,6 @@ struct kbase_hwcnt_csf_physical_layout { * seen, to check any new firmware automatic * samples generated during the watchdog * period. - * @ring_buf: Opaque pointer for ring buffer object. - * @ring_buf_cpu_base: CPU base address of the allocated ring buffer. * @clk_enable_map: The enable map specifying enabled clock domains. * @cycle_count_elapsed: Cycle count elapsed for a given sample period. * @prev_cycle_count: Previous cycle count to calculate the cycle @@ -258,6 +325,13 @@ struct kbase_hwcnt_csf_physical_layout { * is converted from the TIMESTAMP of a dump to CPU MONOTONIC * time. Then it's returned to the caller via 'dump_time_ns' * parameter of dump_get function. + * @last_enable_map: Record of the last requested enable map. + * Used to reapply dump enable request on MCU ON notification. + * @dump_reenable_in_progress: True if backend reenable procedure is in progress. + * @flush_on_disable_in_progress: True when the dump workers are being flushed during dump + * disable procedure. This is needed to prevent the backend + * from being enabled. + * @flush_workers_needed: True when the dump workers need to be flushed. */ struct kbase_hwcnt_backend_csf { struct kbase_hwcnt_backend_csf_info *info; @@ -272,8 +346,6 @@ struct kbase_hwcnt_backend_csf { blk_stt_t *block_states; blk_stt_t *to_user_block_states; u32 watchdog_last_seen_insert_idx; - struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf; - void *ring_buf_cpu_base; u64 clk_enable_map; u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS]; u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS]; @@ -286,8 +358,36 @@ struct kbase_hwcnt_backend_csf { size_t num_l2_slices; u64 powered_shader_core_mask; u64 dump_time_ns; + struct kbase_hwcnt_enable_map last_enable_map; + bool dump_reenable_in_progress; + bool flush_on_disable_in_progress; + bool flush_workers_needed; }; +/* Forward declarations. */ +static void +kbasep_hwcnt_backend_csf_handle_unrecoverable_error(struct kbase_hwcnt_backend_csf *backend_csf); + +static inline bool +kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(enum kbase_hwcnt_backend_csf_enable_state state) +{ + return (state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ) || + (state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK); +} + +static inline bool +kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(enum kbase_hwcnt_backend_csf_enable_state state) +{ + return (state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ) || + (state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK); +} + +static inline bool +kbasep_hwcnt_backend_csf_is_error_state(enum kbase_hwcnt_backend_csf_enable_state state) +{ + return (state == KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER) || + (state == KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR); +} static bool kbasep_hwcnt_backend_csf_backend_exists(struct kbase_hwcnt_backend_csf_info *csf_info) { if (WARN_ON(!csf_info)) @@ -338,7 +438,8 @@ void kbase_hwcnt_backend_csf_set_hw_availability(struct kbase_hwcnt_backend_inte norm_shader_present = virtual_core_mask; } - if (WARN_ON(csf_info->backend->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED)) + /* MCU needs to be powered off. */ + if (WARN_ON(csf_info->mcu_on)) return; if (WARN_ON(num_l2_slices > csf_info->backend->phys_layout.mmu_l2_cnt) || @@ -448,6 +549,7 @@ static void kbasep_hwcnt_backend_csf_init_layout( size_t values_per_block; size_t fw_block_cnt; size_t hw_block_cnt; + size_t md_block_cnt; size_t core_cnt; size_t ne_core_cnt; @@ -458,6 +560,7 @@ static void kbasep_hwcnt_backend_csf_init_layout( values_per_block = prfcnt_info->prfcnt_block_size / KBASE_HWCNT_VALUE_HW_BYTES; fw_block_cnt = div_u64(prfcnt_info->prfcnt_fw_size, prfcnt_info->prfcnt_block_size); hw_block_cnt = div_u64(prfcnt_info->prfcnt_hw_size, prfcnt_info->prfcnt_block_size); + md_block_cnt = div_u64(prfcnt_info->metadata_size, prfcnt_info->prfcnt_block_size); core_cnt = shader_core_cnt; /* In the presence of heterogeneous NE, the SCs that don't have dedicated @@ -479,7 +582,9 @@ static void kbasep_hwcnt_backend_csf_init_layout( .shader_cnt = shader_core_cnt, .fw_block_cnt = fw_block_cnt, .hw_block_cnt = hw_block_cnt, - .block_cnt = fw_block_cnt + hw_block_cnt, + .md_block_cnt = md_block_cnt, + .ringbuf_block_cnt = fw_block_cnt + hw_block_cnt + md_block_cnt, + .payload_block_cnt = fw_block_cnt + hw_block_cnt, .shader_avail_mask = prfcnt_info->sc_core_mask, .headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK, .values_per_block = values_per_block, @@ -493,7 +598,7 @@ static void kbasep_hwcnt_backend_csf_reset_internal_buffers(struct kbase_hwcnt_backend_csf *backend_csf) { size_t user_buf_bytes = backend_csf->info->metadata->dump_buf_bytes; - size_t block_state_bytes = backend_csf->phys_layout.block_cnt * + size_t block_state_bytes = backend_csf->phys_layout.payload_block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; memset(backend_csf->accum_buf, 0, user_buf_bytes); @@ -506,7 +611,7 @@ static void kbasep_hwcnt_backend_csf_reset_consumed_buffers(struct kbase_hwcnt_backend_csf *backend_csf) { size_t user_buf_bytes = backend_csf->info->metadata->dump_buf_bytes; - size_t block_state_bytes = backend_csf->phys_layout.block_cnt * + size_t block_state_bytes = backend_csf->phys_layout.payload_block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; memset(backend_csf->to_user_buf, 0, user_buf_bytes); @@ -523,7 +628,7 @@ kbasep_hwcnt_backend_csf_zero_sample_prfcnt_en_header(struct kbase_hwcnt_backend phys_layout = &backend_csf->phys_layout; - for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { + for (block_idx = 0; block_idx < phys_layout->ringbuf_block_cnt; block_idx++) { block_buf = sample + block_idx * phys_layout->values_per_block; block_buf[phys_layout->enable_mask_offset] = 0; } @@ -537,7 +642,9 @@ kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header(struct kbase_hwcnt_backend_cs char *cpu_dump_base; size_t dump_bytes = backend_csf->info->prfcnt_info.dump_bytes; - cpu_dump_base = (char *)backend_csf->ring_buf_cpu_base; + cpu_dump_base = (char *)backend_csf->info->ring_buf_cpu_base; + if (WARN_ON(!cpu_dump_base)) + return; for (idx = 0; idx < backend_csf->info->ring_buf_cnt; idx++) { sample = (u32 *)&cpu_dump_base[idx * dump_bytes]; @@ -548,18 +655,18 @@ kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header(struct kbase_hwcnt_backend_cs static void kbasep_hwcnt_backend_csf_update_user_sample(struct kbase_hwcnt_backend_csf *backend_csf) { const size_t user_buf_bytes = backend_csf->info->metadata->dump_buf_bytes; - const size_t block_cnt = backend_csf->phys_layout.block_cnt; + const size_t payload_block_cnt = backend_csf->phys_layout.payload_block_cnt; const size_t block_state_bytes = - block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; + payload_block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; size_t i; /* Copy the data into the sample and wait for the user to get it. */ memcpy(backend_csf->to_user_buf, backend_csf->accum_buf, user_buf_bytes); - for (i = 0; i < block_cnt; i++) + for (i = 0; i < payload_block_cnt; i++) kbase_hwcnt_block_state_append(&backend_csf->to_user_block_states[i], backend_csf->block_states[i]); - /* After copied data into user sample, clear the accumulator values to + /* After copying data into user sample, clear the accumulator values to * prepare for the next accumulator, such as the next request or * threshold. */ @@ -568,45 +675,99 @@ static void kbasep_hwcnt_backend_csf_update_user_sample(struct kbase_hwcnt_backe memset(backend_csf->block_states, 0, block_state_bytes); } -void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf *backend, - const u32 enable_mask, bool exiting_protm, - size_t block_idx, blk_stt_t *const block_state, - bool fw_in_protected_mode) +static enum kbase_hwcnt_backend_block_type kbase_hwcnt_backend_csf_identify_block_type( + const struct kbase_hwcnt_csf_physical_layout *phys_layout, size_t block_idx, + size_t *const block_in_type_idx) { - const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend->phys_layout; - /* Offset of shader core blocks from the start of the HW blocks in the sample */ - size_t shader_core_block_offset = - (size_t)(phys_layout->block_cnt - phys_layout->shader_cnt); - bool is_shader_core_block; + const size_t fw_block_offset = phys_layout->md_block_cnt; + const size_t cshw_block_offset = fw_block_offset + phys_layout->fw_block_cnt; + const size_t tiler_block_offset = cshw_block_offset + phys_layout->fe_cnt; + const size_t memsys_block_offset = tiler_block_offset + phys_layout->tiler_cnt; + const size_t shader_core_block_offset = memsys_block_offset + phys_layout->mmu_l2_cnt; + const size_t neural_core_block_offset = shader_core_block_offset + phys_layout->shader_cnt; - size_t neural_core_block_offset = phys_layout->block_cnt - phys_layout->ne_cnt; - bool is_neural_core_block = (block_idx >= neural_core_block_offset); - shader_core_block_offset -= phys_layout->ne_cnt; - is_shader_core_block = - ((block_idx >= shader_core_block_offset) && (block_idx < neural_core_block_offset)); + if (block_idx < fw_block_offset) { + *block_in_type_idx = block_idx; + + return KBASE_HWCNT_BLOCK_METADATA; + } + + if ((fw_block_offset <= block_idx) && (block_idx < cshw_block_offset)) { + *block_in_type_idx = block_idx - fw_block_offset; + + return KBASE_HWCNT_BLOCK_FW; + } + + if ((cshw_block_offset <= block_idx) && (block_idx < tiler_block_offset)) { + *block_in_type_idx = block_idx - cshw_block_offset; + + return KBASE_HWCNT_BLOCK_CSHW; + } + + if ((tiler_block_offset <= block_idx) && (block_idx < memsys_block_offset)) { + *block_in_type_idx = block_idx - tiler_block_offset; + + return KBASE_HWCNT_BLOCK_TILER; + } + + if ((memsys_block_offset <= block_idx) && (block_idx < shader_core_block_offset)) { + *block_in_type_idx = block_idx - memsys_block_offset; + + return KBASE_HWCNT_BLOCK_MEMSYS; + } + if ((shader_core_block_offset <= block_idx) && (block_idx < neural_core_block_offset)) { + *block_in_type_idx = block_idx - shader_core_block_offset; + + return KBASE_HWCNT_BLOCK_SHADER; + } + + if ((neural_core_block_offset <= block_idx) && + (block_idx < phys_layout->ringbuf_block_cnt)) { + *block_in_type_idx = block_idx - neural_core_block_offset; + + return KBASE_HWCNT_BLOCK_NEURAL; + } + + WARN(1, "Unknown block index %zu", block_idx); + *block_in_type_idx = 0; + return KBASE_HWCNT_BLOCK_CSHW; +} + +void kbasep_hwcnt_backend_csf_update_block_state( + struct kbase_hwcnt_backend_csf *backend, const u32 enable_mask, size_t block_idx, + blk_stt_t *const block_state, enum kbase_hwcnt_backend_sample_reason prev_sample_reason, + enum kbase_hwcnt_backend_sample_reason curr_sample_reason) +{ + size_t idx_in_block_type = 0; + enum kbase_hwcnt_backend_block_type type = kbase_hwcnt_backend_csf_identify_block_type( + &backend->phys_layout, block_idx, &idx_in_block_type); + + bool sample_in_protected_mode = (curr_sample_reason == SAMPLE_REASON_BEFORE_PROTM); + bool sample_exiting_protm = (prev_sample_reason == SAMPLE_REASON_BEFORE_PROTM); + bool is_block_on = false; /* Set power bits for the block state for the block, for the sample */ switch (backend->enable_state) { /* Disabled states */ case KBASE_HWCNT_BACKEND_CSF_DISABLED: - case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK: case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: - kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_OFF); + /* Nothing to do */ break; /* Enabled states */ case KBASE_HWCNT_BACKEND_CSF_ENABLED: - case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: - if (!is_shader_core_block) - kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_ON); - else if (!exiting_protm) { + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK: + if (type != KBASE_HWCNT_BLOCK_SHADER) + is_block_on = true; + else if (!sample_exiting_protm) { /* When not exiting protected mode, a zero enable mask on a shader core * counter block indicates the block was powered off for the sample, and * a non-zero counter enable mask indicates the block was powered on for * the sample. */ - kbase_hwcnt_block_state_append(block_state, - (enable_mask ? KBASE_HWCNT_STATE_ON : - KBASE_HWCNT_STATE_OFF)); + is_block_on = !!enable_mask; } break; /* Error states */ @@ -617,6 +778,12 @@ void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf break; } + if (curr_sample_reason == SAMPLE_REASON_BEFORE_SLEEP) + is_block_on = false; + + kbase_hwcnt_block_state_append(block_state, + is_block_on ? KBASE_HWCNT_STATE_ON : KBASE_HWCNT_STATE_OFF); + /* The following four cases apply to a block state in either normal mode or protected mode: * 1. GPU executing in normal mode: Only set normal mode bit. * 2. First sample request after GPU enters protected mode: Set both normal mode and @@ -629,7 +796,7 @@ void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf * 4. Samples requested after GPU exits protected mode: Set both protected mode and normal * mode bits. */ - if (exiting_protm || fw_in_protected_mode) + if (sample_exiting_protm || sample_in_protected_mode) kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_PROTECTED | KBASE_HWCNT_STATE_NORMAL); else @@ -639,30 +806,54 @@ void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf * the shader present and the debug core mask, so explicit checking of the * core mask is not required here. */ - if (is_shader_core_block) { - u64 current_shader_core = 1ULL << (block_idx - shader_core_block_offset); + switch (type) { + case KBASE_HWCNT_BLOCK_METADATA: + case KBASE_HWCNT_BLOCK_FW: + case KBASE_HWCNT_BLOCK_CSHW: + case KBASE_HWCNT_BLOCK_TILER: + case KBASE_HWCNT_BLOCK_MEMSYS: + if (curr_sample_reason == SAMPLE_REASON_BEFORE_YIELD) + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); + else + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE); + break; + case KBASE_HWCNT_BLOCK_SHADER: { + u64 current_shader_core = 1ULL << idx_in_block_type; - WARN_ON_ONCE(backend->phys_layout.shader_cnt > 64); + WARN_ONCE(backend->phys_layout.shader_cnt > 64, + "More than 64 SCs will cause an overflow!"); - if (current_shader_core & backend->info->backend->powered_shader_core_mask) + if (curr_sample_reason == SAMPLE_REASON_BEFORE_YIELD) + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); + else if (current_shader_core & backend->info->backend->powered_shader_core_mask) kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE); else if (current_shader_core & ~backend->info->backend->powered_shader_core_mask) kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); else - WARN_ON_ONCE(true); - } else if (is_neural_core_block) { - u64 current_neural_core = 1ULL << (block_idx - neural_core_block_offset); + WARN_ONCE(true, "Unknown SC availability state!"); + break; + } + case KBASE_HWCNT_BLOCK_NEURAL: { + u64 current_neural_core = 1ULL << idx_in_block_type; - WARN_ON_ONCE(backend->phys_layout.ne_cnt > 64); + WARN_ONCE(backend->phys_layout.ne_cnt > 64, + "More than 64 NE cores may cause an overflow!"); - if (current_neural_core & backend->info->prfcnt_info.ne_core_mask) + if (curr_sample_reason == SAMPLE_REASON_BEFORE_YIELD) + kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); + else if (current_neural_core & backend->info->prfcnt_info.ne_core_mask) kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE); else if (current_neural_core & ~backend->info->prfcnt_info.ne_core_mask) kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_UNAVAILABLE); else - WARN_ON_ONCE(true); - } else - kbase_hwcnt_block_state_append(block_state, KBASE_HWCNT_STATE_AVAILABLE); + WARN_ONCE(true, "Unknown NE core availability state!"); + break; + } + default: + WARN_ONCE(true, "Unknown block type returned (type = %u)", + type); /* This shouldn't happen */ + break; + } } static void kbasep_hwcnt_backend_csf_accumulate_sample(struct kbase_hwcnt_backend_csf *backend, @@ -670,35 +861,40 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample(struct kbase_hwcnt_backen const u32 *new_sample_buf) { const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend->phys_layout; - const size_t dump_bytes = backend->info->prfcnt_info.dump_bytes; + struct kbase_hwcnt_backend_csf_if_prfcnt_info *prfcnt_info = &backend->info->prfcnt_info; + const size_t dump_bytes = prfcnt_info->dump_bytes; const size_t values_per_block = phys_layout->values_per_block; blk_stt_t *const block_states = backend->block_states; - const bool fw_in_protected_mode = backend->info->fw_in_protected_mode; - const bool clearing_samples = backend->info->prfcnt_info.clearing_samples; + enum kbase_hwcnt_backend_sample_reason prev_sample_reason = backend->info->last_transition; + enum kbase_hwcnt_backend_sample_reason curr_sample_reason = SAMPLE_REASON_NONE; + const bool clearing_samples = prfcnt_info->clearing_samples; u64 *accum_buf = backend->accum_buf; size_t block_idx; + const size_t metadata_size = prfcnt_info->metadata_size; const u32 *old_block = old_sample_buf; const u32 *new_block = new_sample_buf; u64 *acc_block = accum_buf; - /* Flag to indicate whether current sample is exiting protected mode. */ - bool exiting_protm = false; - /* The block pointers now point to the first HW block, which is always a CSHW/front-end - * block. The counter enable mask for this block can be checked to determine whether this - * sample is taken after leaving protected mode - this is the only scenario where the CSHW - * block counter enable mask has only the first bit set, and no others. In this case, - * the values in this sample would not be meaningful, so they don't need to be accumulated. - */ - exiting_protm = (new_block[phys_layout->enable_mask_offset] == 1); + size_t num_meta_blocks = prfcnt_info->metadata_size / prfcnt_info->prfcnt_block_size; + size_t num_meta_counters = num_meta_blocks * values_per_block; - for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { + const unsigned int METADATA_SAMPLE_REASON_OFFSET = 3; + + if (backend->phys_layout.md_block_cnt) + curr_sample_reason = new_block[METADATA_SAMPLE_REASON_OFFSET]; + + old_block += num_meta_counters; + new_block += num_meta_counters; + + for (block_idx = 0; block_idx < phys_layout->payload_block_cnt; block_idx++) { const u32 old_enable_mask = old_block[phys_layout->enable_mask_offset]; const u32 new_enable_mask = new_block[phys_layout->enable_mask_offset]; /* Update block state with information of the current sample */ - kbasep_hwcnt_backend_csf_update_block_state(backend, new_enable_mask, exiting_protm, - block_idx, &block_states[block_idx], - fw_in_protected_mode); + kbasep_hwcnt_backend_csf_update_block_state(backend, new_enable_mask, + block_idx + phys_layout->md_block_cnt, + &block_states[block_idx], + prev_sample_reason, curr_sample_reason); if (!(new_enable_mask & HWCNT_BLOCK_EMPTY_SAMPLE)) { /* Hardware block was unavailable or we didn't turn on @@ -728,8 +924,8 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample(struct kbase_hwcnt_backen * request samples without interfering with each other. * * However some early HW may not support wrapping - * counters, for these GPUs counters must be cleared on - * sample to avoid loss of data due to counters + * counters. For these GPUs, counters must be cleared + * on sample to avoid loss of data due to counters * saturating at their maximum value. */ if (!clearing_samples) { @@ -769,8 +965,12 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample(struct kbase_hwcnt_backen } WARN_ON(old_block != old_sample_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES)); WARN_ON(new_block != new_sample_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES)); - WARN_ON(acc_block != accum_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES)); + WARN_ON(acc_block != + accum_buf + ((dump_bytes - metadata_size) / KBASE_HWCNT_VALUE_HW_BYTES)); (void)dump_bytes; + + if (backend->phys_layout.md_block_cnt) + backend->info->last_transition = curr_sample_reason; } static void kbasep_hwcnt_backend_csf_accumulate_samples(struct kbase_hwcnt_backend_csf *backend_csf, @@ -779,33 +979,39 @@ static void kbasep_hwcnt_backend_csf_accumulate_samples(struct kbase_hwcnt_backe { u32 raw_idx; unsigned long flags = 0UL; - u8 *cpu_dump_base = (u8 *)backend_csf->ring_buf_cpu_base; + u8 *cpu_dump_base = (u8 *)backend_csf->info->ring_buf_cpu_base; const size_t ring_buf_cnt = backend_csf->info->ring_buf_cnt; const size_t buf_dump_bytes = backend_csf->info->prfcnt_info.dump_bytes; u32 *old_sample_buf = backend_csf->old_sample_buf; u32 *new_sample_buf = old_sample_buf; - const struct kbase_hwcnt_csf_physical_layout *phys_layout = &backend_csf->phys_layout; if (extract_index_to_start == insert_index_to_stop) { /* No samples to accumulate but block states need to be updated for dump. */ size_t block_idx; - for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { + /* We cannot use the last sample reason here, because the sequencing with the + * previous dump is not entirely clear and the required flag may not be set + * at that time. + */ + blk_stt_t stt = backend_csf->info->fw_in_protected_mode ? + KBASE_HWCNT_STATE_PROTECTED : + KBASE_HWCNT_STATE_NORMAL; + + for (block_idx = 0; block_idx < backend_csf->phys_layout.payload_block_cnt; + block_idx++) { /* Set protected mode bit for block state if GPU is in protected mode, * otherwise set the normal mode bit. */ - kbase_hwcnt_block_state_append(&backend_csf->block_states[block_idx], - backend_csf->info->fw_in_protected_mode ? - KBASE_HWCNT_STATE_PROTECTED : - KBASE_HWCNT_STATE_NORMAL); + kbase_hwcnt_block_state_append(&backend_csf->block_states[block_idx], stt); } return; } /* Sync all the buffers to CPU side before read the data. */ backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf, extract_index_to_start, - insert_index_to_stop, true); + backend_csf->info->ring_buf, + extract_index_to_start, insert_index_to_stop, + true); /* Consider u32 wrap case, '!=' is used here instead of '<' operator */ for (raw_idx = extract_index_to_start; raw_idx != insert_index_to_stop; raw_idx++) { @@ -834,8 +1040,9 @@ static void kbasep_hwcnt_backend_csf_accumulate_samples(struct kbase_hwcnt_backe /* Sync zeroed buffers to avoid coherency issues on future use. */ backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf, extract_index_to_start, - insert_index_to_stop, false); + backend_csf->info->ring_buf, + extract_index_to_start, insert_index_to_stop, + false); /* After consuming all samples between extract_idx and insert_idx, * set the raw extract index to insert_idx so that the sample buffers @@ -858,6 +1065,7 @@ static void kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( backend_csf->info->csf_if->assert_lock_held(backend_csf->info->csf_if->ctx); if (backend_csf->enable_state != new_state) { + pr_debug("%s %d -> %d", __func__, backend_csf->enable_state, new_state); backend_csf->enable_state = new_state; wake_up(&backend_csf->enable_state_waitq); @@ -872,7 +1080,8 @@ static void kbasep_hwcnt_backend_watchdog_timer_cb(void *info) csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); - if (WARN_ON(!kbasep_hwcnt_backend_csf_backend_exists(csf_info))) { + /* Early-exit if backend doesn't exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); return; } @@ -882,9 +1091,11 @@ static void kbasep_hwcnt_backend_watchdog_timer_cb(void *info) /* Only do watchdog request when all conditions are met: */ if (/* 1. Backend is enabled. */ (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) && - /* 2. FW is not in protected mode. */ + /* 2. MCU is ON. */ + csf_info->mcu_on && + /* 3. FW is not in protected mode. */ (!csf_info->fw_in_protected_mode) && - /* 3. dump state indicates no other dumping is in progress. */ + /* 4. dump state indicates no other dumping is in progress. */ ((backend_csf->dump_state == KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE) || (backend_csf->dump_state == KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED))) { u32 extract_index = 0U; @@ -905,13 +1116,19 @@ static void kbasep_hwcnt_backend_watchdog_timer_cb(void *info) } } - /* Must schedule another callback when in the transitional state because - * this function can be called for the first time before the performance - * counter enabled interrupt. + /* Reschedule the next callback when watchdog is enabled (i.e. backend states ENABLED + * and KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK). + * + * Keep in mind that this function can be called for the first time + * before the performance counter enabled interrupt. + * + * Do not reschedule if MCU is off. */ - if ((backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) || - (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED)) { - /* Reschedule the timer for next watchdog callback. */ + if (((backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) || + (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK)) && + csf_info->mcu_on) { + /* Reschedule the timer for next watchdog callback.*/ csf_info->watchdog_if->modify(csf_info->watchdog_if->timer, csf_info->watchdog_timer_interval_ms); } @@ -935,7 +1152,9 @@ static void kbasep_hwcnt_backend_csf_dump_worker(struct work_struct *work) u32 extract_index = 0U; u32 insert_index = 0U; u64 ts_gpu, ts_dump_raw, ts_dump, ts_dump_request, ts_now; + u32 *ts_gpu_ptr; bool sample_accumulated; + size_t ts_offset_in_sample_buf; WARN_ON(!work); backend_csf = container_of(work, struct kbase_hwcnt_backend_csf, hwc_dump_work); @@ -982,7 +1201,11 @@ static void kbasep_hwcnt_backend_csf_dump_worker(struct work_struct *work) WARN_ON(backend_csf != backend_csf->info->backend); /* TIMESTAMP_LO/HI of the newest dump */ - ts_gpu = *(u64 *)backend_csf->old_sample_buf; + ts_offset_in_sample_buf = (backend_csf->phys_layout.md_block_cnt * + backend_csf->info->prfcnt_info.prfcnt_block_size) / + KBASE_HWCNT_VALUE_HW_BYTES; + ts_gpu_ptr = backend_csf->old_sample_buf + ts_offset_in_sample_buf; + ts_gpu = *(u64 *)ts_gpu_ptr; /* Convert TIMESTAMP_LO/HI of a newest dump into a CPU timestamp. */ ts_dump_raw = backend_csf->info->csf_if->time_convert_gpu_to_cpu( @@ -1170,17 +1393,53 @@ kbasep_hwcnt_backend_csf_append_block_states(struct kbase_hwcnt_backend_csf *bac { size_t i; - for (i = 0; i < backend_csf->phys_layout.block_cnt; i++) + for (i = 0; i < backend_csf->phys_layout.payload_block_cnt; i++) kbase_hwcnt_block_state_append(&backend_csf->to_user_block_states[i], block_state); } +static int kbasep_hwcnt_backend_csf_do_dump_enable(struct kbase_hwcnt_backend_csf *backend_csf, + const struct kbase_hwcnt_enable_map *enable_map) +{ + struct kbase_hwcnt_backend_csf_if_enable enable; + int err; + + if (!backend_csf || !enable_map || (enable_map->metadata != backend_csf->info->metadata) || + WARN_ON(!backend_csf->info->ring_buf)) + return -EINVAL; + + backend_csf->info->csf_if->assert_lock_held(backend_csf->info->csf_if->ctx); + WARN_ON(backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ); + + if (!backend_csf->info->mcu_on) + return 0; + + err = backend_csf->info->watchdog_if->enable(backend_csf->info->watchdog_if->timer, + backend_csf->info->watchdog_timer_interval_ms, + kbasep_hwcnt_backend_watchdog_timer_cb, + backend_csf->info); + if (err) + return err; + + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK); + + kbasep_hwcnt_backend_csf_get_physical_enable(backend_csf, enable_map, &enable); + + backend_csf->info->csf_if->dump_enable(backend_csf->info->csf_if->ctx, + backend_csf->info->ring_buf, &enable); + kbasep_hwcnt_backend_csf_cc_initial_sample(backend_csf, enable_map); + + return 0; +} + /* CSF backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ static int kbasep_hwcnt_backend_csf_dump_enable_nolock(struct kbase_hwcnt_backend *backend, const struct kbase_hwcnt_enable_map *enable_map) { struct kbase_hwcnt_backend_csf *backend_csf = (struct kbase_hwcnt_backend_csf *)backend; - struct kbase_hwcnt_backend_csf_if_enable enable; + int err; if (!backend_csf || !enable_map || (enable_map->metadata != backend_csf->info->metadata)) @@ -1200,28 +1459,35 @@ kbasep_hwcnt_backend_csf_dump_enable_nolock(struct kbase_hwcnt_backend *backend, */ kbasep_hwcnt_backend_csf_append_block_states(backend_csf, KBASE_HWCNT_STATE_OFF); - kbasep_hwcnt_backend_csf_get_physical_enable(backend_csf, enable_map, &enable); - - /* enable_state should be DISABLED before we transfer it to enabled */ - if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED) + /* enable_state should be (or be transitioning to) DISABLED before we transfer it + * to enabled. In addition to that, there shouldn't be a concurrent dump worker flush during + * dump_disable() procedure (the lock is released then). + */ + if (backend_csf->flush_on_disable_in_progress || + (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED && + !kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(backend_csf->enable_state))) return -EIO; - err = backend_csf->info->watchdog_if->enable(backend_csf->info->watchdog_if->timer, - backend_csf->info->watchdog_timer_interval_ms, - kbasep_hwcnt_backend_watchdog_timer_cb, - backend_csf->info); - if (err) - return err; - backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; WARN_ON(!completion_done(&backend_csf->dump_completed)); + + /* Save the enable_map. It will be used e.g. when handling the deferred enable REQ + * or during cold_boot procedure. + */ + kbase_hwcnt_enable_map_copy(&backend_csf->last_enable_map, enable_map); + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED); + backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ); - backend_csf->info->csf_if->dump_enable(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf, &enable); - - kbasep_hwcnt_backend_csf_cc_initial_sample(backend_csf, enable_map); + /* If MCU is powered off, backend will stay in TRANSITIONING_TO_ENABLED_PENDING_REQ. + * Enable REQ will be sent during MCU ON notification. + */ + if (backend_csf->info->mcu_on) { + err = kbasep_hwcnt_backend_csf_do_dump_enable(backend_csf, + &backend_csf->last_enable_map); + if (err) + return err; + } return 0; } @@ -1243,25 +1509,60 @@ static int kbasep_hwcnt_backend_csf_dump_enable(struct kbase_hwcnt_backend *back return errcode; } -static void kbasep_hwcnt_backend_csf_wait_enable_transition_complete( - struct kbase_hwcnt_backend_csf *backend_csf, unsigned long *lock_flags) +static void +kbasep_hwcnt_backend_csf_wait_state_transition_complete(struct kbase_hwcnt_backend_csf *backend_csf, + unsigned long *lock_flags) { backend_csf->info->csf_if->assert_lock_held(backend_csf->info->csf_if->ctx); - while ((backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) || - (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED)) { + /* The wait loop ends when backend gets out of a transitioning state or MCU + * is powered off. + */ + while ((kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(backend_csf->enable_state) || + kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(backend_csf->enable_state)) && + backend_csf->info->mcu_on) { backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, *lock_flags); wait_event(backend_csf->enable_state_waitq, - (backend_csf->enable_state != - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) && - (backend_csf->enable_state != - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED)); + (!kbasep_hwcnt_backend_csf_is_state_trans_to_enabled( + backend_csf->enable_state) && + !kbasep_hwcnt_backend_csf_is_state_trans_to_disabled( + backend_csf->enable_state)) || + !backend_csf->info->mcu_on); backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, lock_flags); } } +static void kbasep_hwcnt_backend_csf_send_disable_req(struct kbase_hwcnt_backend_csf_info *csf_info) +{ + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + + /* Send the disable REQ and mark that we don't have any pending disable REQ. */ + csf_info->csf_if->dump_disable(csf_info->csf_if->ctx); + csf_info->pending_disable_req = false; +} + +static void +kbasep_hwcnt_backend_csf_resolve_wait_for_worker_state(struct kbase_hwcnt_backend_csf *backend_csf) +{ + backend_csf->info->csf_if->assert_lock_held(backend_csf->info->csf_if->ctx); + + switch (backend_csf->enable_state) { + case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED); + break; + case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR); + break; + default: + /* Do nothing */ + break; + } +} + /* CSF backend implementation of kbase_hwcnt_backend_dump_disable_fn */ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *backend, struct kbase_hwcnt_dump_buffer *dump_buffer, @@ -1277,12 +1578,19 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba (dump_buffer && !enable_map))) return; + /* Cancel any pending reenable worker so we can disable the backend cleanly. */ + cancel_work_sync(&backend_csf->info->hwc_dump_reenable_work); + backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); /* Make sure we wait until any previous enable or disable have completed * before doing anything. + * + * Note: Backend might still be in a transitioning state after + * kbasep_hwcnt_backend_csf_wait_state_transition_complete() if MCU was + * powered off. */ - kbasep_hwcnt_backend_csf_wait_enable_transition_complete(backend_csf, &flags); + kbasep_hwcnt_backend_csf_wait_state_transition_complete(backend_csf, &flags); if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED || backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) { @@ -1293,27 +1601,36 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba return; } - if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) { + /* Send disable REQ if the backend was previously either enabled or was transitioning + * to enabled state but it was never REQed/ACKed due to MCU being powered off. + * + * In any other state the disable REQ have already been sent. + */ + if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED || + kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(backend_csf->enable_state)) { kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); + backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ); backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; complete_all(&backend_csf->dump_completed); - /* Only disable if we were previously enabled - in all other - * cases the call to disable will have already been made. - */ do_disable = true; } WARN_ON(backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE); WARN_ON(!completion_done(&backend_csf->dump_completed)); + /* Mark that we are going to flush the workers (hence we are releasing the lock). + * This is needed to prevent any potential concurrent attempts to enable the backend. + */ + backend_csf->flush_on_disable_in_progress = true; + backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); /* Deregister the timer and block until any timer callback has completed. * We've transitioned out of the ENABLED state so we can guarantee it * won't reschedule itself. */ - backend_csf->info->watchdog_if->disable(backend_csf->info->watchdog_if->timer); + backend_csf->info->watchdog_if->disable(backend_csf->info->watchdog_if->timer, + KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_BLOCK); /* Block until any async work has completed. We have transitioned out of * the ENABLED state so we can guarantee no new work will concurrently @@ -1322,26 +1639,49 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba flush_workqueue(backend_csf->hwc_dump_workq); backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); + backend_csf->flush_on_disable_in_progress = false; - if (do_disable) - backend_csf->info->csf_if->dump_disable(backend_csf->info->csf_if->ctx); - - kbasep_hwcnt_backend_csf_wait_enable_transition_complete(backend_csf, &flags); - - switch (backend_csf->enable_state) { - case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: - kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED); - break; - case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: - kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR); - break; - default: - WARN_ON(true); - break; + if (!backend_csf->info->mcu_on) { + /* Stay in TRANSITIONING_TO_DISABLED_PENDING_REQ state. + * Record that we wanted to disable HWC but the disable REQ hasn't reached MCU. + * We need to save this in a backend instance-independent structure, + * to be able to send the disable REQ at MCU ON notification even if the backend + * has been destroyed/recreated. + */ + backend_csf->info->pending_disable_req = true; + goto skip_disable_req; } + if (do_disable) { + /* Check if we have entered an error state while we released the lock. + * If yes, don't leave the error state. + */ + if (!kbasep_hwcnt_backend_csf_is_error_state(backend_csf->enable_state)) + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK); + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); + } + + if (!kbasep_hwcnt_backend_csf_is_error_state(backend_csf->enable_state)) + kbasep_hwcnt_backend_csf_wait_state_transition_complete(backend_csf, &flags); + + kbasep_hwcnt_backend_csf_resolve_wait_for_worker_state(backend_csf); + + /* At this point the correct states are : + * - KBASE_HWCNT_BACKEND_CSF_DISABLED + * - KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR + * - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK (only if MCU + * is powered off). + */ + + WARN_ON((backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED) && + (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) && + ((backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK) || + backend_csf->info->mcu_on)); + +skip_disable_req: backend_csf->user_requested = false; backend_csf->watchdog_last_seen_insert_idx = 0; @@ -1354,7 +1694,7 @@ static void kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *ba /* Sync zeroed buffers to avoid coherency issues on future use. */ backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf, 0, + backend_csf->info->ring_buf, 0, backend_csf->info->ring_buf_cnt, false); /* Disabling HWCNT is an indication that blocks have been powered off. This is important to @@ -1408,12 +1748,19 @@ static int kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *bac return -EINVAL; backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); - - /* If we're transitioning to enabled there's nothing to accumulate, and + /* If we're transitioning to enabled, then there's nothing to accumulate, and * the user dump buffer is already zeroed. We can just short circuit to * the DUMP_COMPLETED state. + * + * We can also early exit in case of an ongoing reenable procedure + * (dump_reenable_in_progress). It means that we must have disabled the backend + * first so there is nothing to accumulate. */ - if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) { + if (kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(backend_csf->enable_state) || + (backend_csf->dump_reenable_in_progress && + ((backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK) || + (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED)))) { backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED; backend_csf->dump_time_ns = kbasep_hwcnt_backend_csf_timestamp_ns(backend); kbasep_hwcnt_backend_csf_cc_update(backend_csf); @@ -1421,12 +1768,11 @@ static int kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *bac * during multiple dumps, hence append the OFF state. */ kbasep_hwcnt_backend_csf_append_block_states(backend_csf, KBASE_HWCNT_STATE_OFF); - backend_csf->user_requested = true; + WARN_ON(!completion_done(&backend_csf->dump_completed)); backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); return 0; } - /* Otherwise, make sure we're already enabled. */ if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_ENABLED) { backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); @@ -1461,9 +1807,9 @@ static int kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *bac watchdog_dumping = true; if ((backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) && - !backend_csf->info->fw_in_protected_mode) { - /* Only do the request if we are fully enabled and not in - * protected mode. + !backend_csf->info->fw_in_protected_mode && backend_csf->info->mcu_on) { + /* Do the request only if the MCU is on, we are fully enabled + * and not in protected mode. */ backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED; do_request = true; @@ -1496,13 +1842,16 @@ static int kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *bac } else kbase_hwcnt_backend_csf_submit_dump_worker(backend_csf->info); - backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); - /* Modify watchdog timer to delay the regular check time since - * just requested. + * just requested (only if MCU is ON). */ - backend_csf->info->watchdog_if->modify(backend_csf->info->watchdog_if->timer, - backend_csf->info->watchdog_timer_interval_ms); + if (backend_csf->info->mcu_on) { + backend_csf->info->watchdog_if->modify( + backend_csf->info->watchdog_if->timer, + backend_csf->info->watchdog_timer_interval_ms); + } + + backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); return 0; } @@ -1613,8 +1962,7 @@ static void kbasep_hwcnt_backend_csf_destroy(struct kbase_hwcnt_backend_csf *bac destroy_workqueue(backend_csf->hwc_dump_workq); - backend_csf->info->csf_if->ring_buf_free(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf); + kbase_hwcnt_enable_map_free(&backend_csf->last_enable_map); kfree(backend_csf->accum_buf); backend_csf->accum_buf = NULL; @@ -1650,8 +1998,8 @@ static int kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info * int errcode = -ENOMEM; size_t block_state_bytes; - WARN_ON(!csf_info); - WARN_ON(!out_backend); + if (WARN_ON(!out_backend || !csf_info || !csf_info->ring_buf)) + return -EINVAL; backend_csf = kzalloc(sizeof(*backend_csf), GFP_KERNEL); if (!backend_csf) @@ -1674,8 +2022,8 @@ static int kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info * goto err_alloc_user_sample_buf; /* Allocate space to store block state values for each block */ - block_state_bytes = backend_csf->phys_layout.block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * - KBASE_HWCNT_BLOCK_STATE_STRIDE; + block_state_bytes = backend_csf->phys_layout.payload_block_cnt * + KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; backend_csf->block_states = kzalloc(block_state_bytes, GFP_KERNEL); if (!backend_csf->block_states) goto err_alloc_block_states_buf; @@ -1684,11 +2032,14 @@ static int kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info * if (!backend_csf->to_user_block_states) goto err_alloc_user_block_state_buf; - errcode = csf_info->csf_if->ring_buf_alloc(csf_info->csf_if->ctx, csf_info->ring_buf_cnt, - &backend_csf->ring_buf_cpu_base, - &backend_csf->ring_buf); + /* Allocate a local version of the enable_map to avoid a situation where the backend's + * caller frees the map when it's still required for the backend (e.g. enable_map + * being reapplied after MCU power on). + */ + errcode = kbase_hwcnt_enable_map_alloc(csf_info->metadata, &backend_csf->last_enable_map); if (errcode) - goto err_ring_buf_alloc; + goto err_alloc_last_enable_map; + errcode = -ENOMEM; /* Zero all performance enable header to prepare for first enable. */ @@ -1696,7 +2047,7 @@ static int kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info * /* Sync zeroed buffers to avoid coherency issues on use. */ backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf, 0, + backend_csf->info->ring_buf, 0, backend_csf->info->ring_buf_cnt, false); init_completion(&backend_csf->dump_completed); @@ -1724,9 +2075,8 @@ static int kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info * return 0; err_alloc_workqueue: - backend_csf->info->csf_if->ring_buf_free(backend_csf->info->csf_if->ctx, - backend_csf->ring_buf); -err_ring_buf_alloc: + kbase_hwcnt_enable_map_free(&backend_csf->last_enable_map); +err_alloc_last_enable_map: kfree(backend_csf->to_user_block_states); backend_csf->to_user_block_states = NULL; err_alloc_user_block_state_buf: @@ -1747,6 +2097,85 @@ alloc_error: return errcode; } +enum kbasep_hwcnt_backend_ring_buf_free_type { + KBASEP_RING_BUF_FREE_NORMAL, + KBASEP_RING_BUF_FREE_DEFERRED, +}; + +static inline void +kbasep_hwcnt_backend_csf_ring_buf_free_deferred(struct kbase_hwcnt_backend_csf_info *csf_info) +{ + if (WARN_ON(!csf_info)) + return; + + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + + csf_info->ring_buf_cleanup = csf_info->ring_buf; + queue_work(csf_info->hwc_deferred_op_workq, &csf_info->hwc_ring_buf_free_work); + csf_info->ring_buf = NULL; +} + +static inline void kbasep_hwcnt_backend_csf_ring_buf_refcnt_increment_nolock( + struct kbase_hwcnt_backend_csf_info *csf_info) +{ + if (WARN_ON(!csf_info)) + return; + + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + + csf_info->ring_buf_refcnt.val++; + WARN_ON(csf_info->ring_buf_refcnt.val > RING_BUF_REFCNT_MAX); +} + +static void kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement_nolock( + struct kbase_hwcnt_backend_csf_info *csf_info, unsigned long *lock_flags, + enum kbasep_hwcnt_backend_ring_buf_free_type type) +{ + if (WARN_ON(!csf_info)) + return; + + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + + csf_info->ring_buf_refcnt.val--; + WARN_ON(csf_info->ring_buf_refcnt.val < RING_BUF_REFCNT_MIN); + + if (!csf_info->ring_buf_refcnt.val) { + struct kbase_hwcnt_backend_csf_if_ring_buf *tmp_ring_buf = csf_info->ring_buf; + + WARN_ON(!csf_info->ring_buf); + + switch (type) { + case KBASEP_RING_BUF_FREE_NORMAL: + if (WARN_ON(!lock_flags)) + break; + csf_info->ring_buf = NULL; + csf_info->csf_if->unlock(csf_info->csf_if->ctx, *lock_flags); + csf_info->csf_if->ring_buf_free(csf_info->csf_if->ctx, tmp_ring_buf); + csf_info->csf_if->lock(csf_info->csf_if->ctx, lock_flags); + break; + case KBASEP_RING_BUF_FREE_DEFERRED: + kbasep_hwcnt_backend_csf_ring_buf_free_deferred(csf_info); + break; + default: + break; + } + } +} + +static void kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement( + struct kbase_hwcnt_backend_csf_info *csf_info, + enum kbasep_hwcnt_backend_ring_buf_free_type type) +{ + unsigned long flags = 0UL; + + if (WARN_ON(!csf_info)) + return; + + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement_nolock(csf_info, &flags, type); + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + /* CSF backend implementation of kbase_hwcnt_backend_init_fn */ static int kbasep_hwcnt_backend_csf_init(const struct kbase_hwcnt_backend_info *info, struct kbase_hwcnt_backend **out_backend) @@ -1760,10 +2189,35 @@ static int kbasep_hwcnt_backend_csf_init(const struct kbase_hwcnt_backend_info * if (!info || !out_backend) return -EINVAL; + /* Ring buffer allocation. */ + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + kbasep_hwcnt_backend_csf_ring_buf_refcnt_increment_nolock(csf_info); + if (csf_info->ring_buf_refcnt.val == 1) { + WARN_ON(csf_info->ring_buf); + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); + + /* Flush any pending ring_buf_free_worker before allocating a new ring buffer. */ + flush_workqueue(csf_info->hwc_deferred_op_workq); + + errcode = csf_info->csf_if->ring_buf_alloc(csf_info->csf_if->ctx, + csf_info->ring_buf_cnt, + &csf_info->ring_buf_cpu_base, + &csf_info->ring_buf); + if (errcode) { + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement( + csf_info, KBASEP_RING_BUF_FREE_NORMAL); + return errcode; + } + } else + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); + /* Create the backend. */ errcode = kbasep_hwcnt_backend_csf_create(csf_info, &backend_csf); - if (errcode) + if (errcode) { + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement(csf_info, + KBASEP_RING_BUF_FREE_NORMAL); return errcode; + } /* If it was not created before, attach it to csf_info. * Use spin lock to avoid concurrent initialization. @@ -1783,6 +2237,8 @@ static int kbasep_hwcnt_backend_csf_init(const struct kbase_hwcnt_backend_info * * function properly. */ if (!success) { + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement(backend_csf->info, + KBASEP_RING_BUF_FREE_NORMAL); kbasep_hwcnt_backend_csf_destroy(backend_csf); return -EBUSY; } @@ -1806,6 +2262,8 @@ static void kbasep_hwcnt_backend_csf_term(struct kbase_hwcnt_backend *backend) */ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); backend_csf->info->backend = NULL; + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement_nolock(backend_csf->info, &flags, + KBASEP_RING_BUF_FREE_NORMAL); backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, flags); kbasep_hwcnt_backend_csf_destroy(backend_csf); @@ -1827,6 +2285,141 @@ static void kbasep_hwcnt_backend_csf_release(const struct kbase_hwcnt_backend *b csf_info->csf_if->release(csf_info->csf_if->ctx); } +static inline bool +kbasep_hwcnt_backend_csf_abort_dump_reenable(struct kbase_hwcnt_backend_csf_info *csf_info) +{ + if (WARN_ON(!csf_info)) + return true; + + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + /* Check if the dump reenabling is still valid. + * + * dump_reenable_in_progress can be set to false by a cold_boot procedure, + * which is a signal to abort the reenable work. + */ + return (!kbasep_hwcnt_backend_csf_backend_exists(csf_info) || + !csf_info->backend->dump_reenable_in_progress || + (csf_info->backend->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK && + csf_info->backend->enable_state != + KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER && + csf_info->backend->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED)); +} + +/** + * kbasep_hwcnt_backend_csf_dump_reenable_worker() - HWC backend reenable worker. + * + * @work: Work structure. + * + * Called when a HWC backend needs a disable-enable cycle, e.g. when we want to enable the backend, + * but the previous disable hasn't been performed due to MCU being powered off. + * + * This procedure assumes that the disable REQ has been already sent in the atomic context, + * and this worker waits for the disable ACK, and later sends an enable REQ. + */ +static void kbasep_hwcnt_backend_csf_dump_reenable_worker(struct work_struct *work) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + unsigned long flags = 0ULL; + int err; + + WARN_ON(!work); + + csf_info = container_of(work, struct kbase_hwcnt_backend_csf_info, hwc_dump_reenable_work); + + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + /* Check if the re-enabling is still valid. + * + * The thread that enqueued this worker has already sent the disable REQ, so we either wait + * for the disable ACK, or it was already ACKed. Any other state (or a destroyed backend) + * means that the re-enabling should be aborted. + */ + if (kbasep_hwcnt_backend_csf_abort_dump_reenable(csf_info)) + goto exit; + + /* Wait for the disable ACK. */ + kbasep_hwcnt_backend_csf_wait_state_transition_complete(csf_info->backend, &flags); + + /* kbasep_hwcnt_backend_csf_wait_state_transition_complete releases the lock, so we need to + * recheck if reenabling is still valid. + */ + if (kbasep_hwcnt_backend_csf_abort_dump_reenable(csf_info)) + goto exit; + + /* We are re-enabling the backend, so we must have already waited for the workers + * to complete while disabling the backend. It is safe to transition out of + * *_WAIT_FOR_WORKER states. + */ + kbasep_hwcnt_backend_csf_resolve_wait_for_worker_state(csf_info->backend); + + if (csf_info->backend->enable_state == KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) + goto exit; + + /* The disabling part of the worker has concluded. + * + * Note that it is fine to be in TRANSITIONING_TO_DISABLED_PENDING_ACK at this point. + * It means that we have already sent disable REQ while the MCU was on, but the MCU + * was powered off while kbasep_hwcnt_backend_csf_wait_state_transition_complete was called + * previously. + * + * We can proceed with the re-enabling procedure because: + * - the pending disable will be ACKed at the following MCU power on, + * - enable REQ will be sent at the next MCU power on, because this reenable worker + * will set the TRANSITIONING_TO_ENABLED_PENDING_REQ state. + */ + WARN_ON(!(csf_info->backend->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED || + csf_info->backend->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK)); + + /* Proceed to enabling part of the worker. */ + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + csf_info->backend, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ); + + /* If MCU is powered off, backend will stay in TRANSITIONING_TO_ENABLED_PENDING_REQ. + * Enable REQ will be sent during MCU ON notification. + */ + if (csf_info->mcu_on) { + err = kbasep_hwcnt_backend_csf_do_dump_enable(csf_info->backend, + &csf_info->backend->last_enable_map); + if (err) + kbasep_hwcnt_backend_csf_handle_unrecoverable_error(csf_info->backend); + } + +exit: + if (kbasep_hwcnt_backend_csf_backend_exists(csf_info)) + csf_info->backend->dump_reenable_in_progress = false; + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + +/** + * kbasep_hwcnt_backend_csf_ring_buf_free_worker() - Ring buffer freeing worker. + * + * @work: Work structure. + * + * Called when a ring buffer needs to be freed. + */ +static void kbasep_hwcnt_backend_csf_ring_buf_free_worker(struct work_struct *work) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf; + unsigned long flags = 0UL; + + WARN_ON(!work); + + csf_info = container_of(work, struct kbase_hwcnt_backend_csf_info, hwc_ring_buf_free_work); + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + + if (WARN_ON(!csf_info->ring_buf_cleanup)) { + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); + return; + } + ring_buf = csf_info->ring_buf_cleanup; + csf_info->ring_buf_cleanup = NULL; + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); + + csf_info->csf_if->ring_buf_free(csf_info->csf_if->ctx, ring_buf); +} + /** * kbasep_hwcnt_backend_csf_info_destroy() - Destroy a CSF backend info. * @info: Pointer to info to destroy. @@ -1839,12 +2432,18 @@ static void kbasep_hwcnt_backend_csf_info_destroy(const struct kbase_hwcnt_backe if (!info) return; + destroy_workqueue(info->hwc_deferred_op_workq); + /* The backend should be destroyed before the info object destroy. */ WARN_ON(info->backend != NULL); /* The metadata should be destroyed before the info object destroy. */ WARN_ON(info->metadata != NULL); + /* Ring buffer should be freed either during backend destruction or device termination.*/ + WARN_ON(info->ring_buf != NULL); + WARN_ON(info->ring_buf_cleanup != NULL); + kfree(info); } @@ -1892,7 +2491,19 @@ kbasep_hwcnt_backend_csf_info_create(struct kbase_hwcnt_backend_csf_if *csf_if, .fw_in_protected_mode = false, .unrecoverable_error_happened = false, .watchdog_if = watchdog_if, .watchdog_timer_interval_ms = watchdog_timer_interval_ms, + .last_transition = SAMPLE_REASON_NONE, .mcu_on = false, }; + + info->hwc_deferred_op_workq = + alloc_workqueue("mali_hwc_deferred_op_wq", WQ_HIGHPRI | WQ_UNBOUND, 1); + if (!info->hwc_deferred_op_workq) { + kfree(info); + return -ENOMEM; + } + + INIT_WORK(&info->hwc_dump_reenable_work, kbasep_hwcnt_backend_csf_dump_reenable_worker); + INIT_WORK(&info->hwc_ring_buf_free_work, kbasep_hwcnt_backend_csf_ring_buf_free_worker); + *out_info = info; return 0; @@ -1913,10 +2524,13 @@ kbasep_hwcnt_backend_csf_metadata(const struct kbase_hwcnt_backend_info *info) static void kbasep_hwcnt_backend_csf_handle_unrecoverable_error(struct kbase_hwcnt_backend_csf *backend_csf) { - bool do_disable = false; + bool send_disable_req = false; backend_csf->info->csf_if->assert_lock_held(backend_csf->info->csf_if->ctx); + /* Cancel any pending backend reenabling procedure. */ + backend_csf->dump_reenable_in_progress = false; + /* We are already in or transitioning to the unrecoverable error state. * Early out. */ @@ -1934,12 +2548,12 @@ kbasep_hwcnt_backend_csf_handle_unrecoverable_error(struct kbase_hwcnt_backend_c return; } - /* Trigger a disable only if we are not already transitioning to - * disabled, we don't want to disable twice if an unrecoverable error + /* Send a disable REQ only if we are not already transitioning to + * disabled. We don't want to send disable REQ twice if an unrecoverable error * happens while we are disabling. */ - do_disable = - (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); + send_disable_req = + !kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(backend_csf->enable_state); kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( backend_csf, KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER); @@ -1950,12 +2564,8 @@ kbasep_hwcnt_backend_csf_handle_unrecoverable_error(struct kbase_hwcnt_backend_c backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; complete_all(&backend_csf->dump_completed); - /* Trigger a disable only if we are not already transitioning to - * disabled, - we don't want to disable twice if an unrecoverable error - * happens while we are disabling. - */ - if (do_disable) - backend_csf->info->csf_if->dump_disable(backend_csf->info->csf_if->ctx); + if (send_disable_req) + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); } static void @@ -1966,14 +2576,16 @@ kbasep_hwcnt_backend_csf_handle_recoverable_error(struct kbase_hwcnt_backend_csf switch (backend_csf->enable_state) { case KBASE_HWCNT_BACKEND_CSF_DISABLED: case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: - case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK: case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR: case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: /* Already disabled or disabling, or in an unrecoverable error. * Nothing to be done to handle the error. */ return; - case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ: + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK: /* A seemingly recoverable error that occurs while we are * transitioning to enabled is probably unrecoverable. */ @@ -1986,14 +2598,15 @@ kbasep_hwcnt_backend_csf_handle_recoverable_error(struct kbase_hwcnt_backend_csf * disable(). */ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); - /* Transition the dump to the IDLE state and unblock any - * waiters. The IDLE state signifies an error. + backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK); + /* Transition the dump to the IDLE state and unblock any waiters. The IDLE state + * signifies an error. Mark that dump workers need to be flushed. */ + backend_csf->flush_workers_needed = true; backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; complete_all(&backend_csf->dump_completed); - backend_csf->info->csf_if->dump_disable(backend_csf->info->csf_if->ctx); + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); return; } } @@ -2015,7 +2628,6 @@ void kbase_hwcnt_backend_csf_protm_entered(struct kbase_hwcnt_backend_interface void kbase_hwcnt_backend_csf_protm_exited(struct kbase_hwcnt_backend_interface *iface) { struct kbase_hwcnt_backend_csf_info *csf_info; - csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); @@ -2057,6 +2669,7 @@ void kbase_hwcnt_backend_csf_on_before_reset(struct kbase_hwcnt_backend_interfac csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); return; } + backend_csf = csf_info->backend; if ((backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED) && @@ -2164,15 +2777,22 @@ void kbase_hwcnt_backend_csf_on_prfcnt_enable(struct kbase_hwcnt_backend_interfa /* Early out if the backend does not exist. */ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) return; + backend_csf = csf_info->backend; - if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) { + if (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK) { kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( backend_csf, KBASE_HWCNT_BACKEND_CSF_ENABLED); } else if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) { /* Unexpected, but we are already in the right state so just * ignore it. */ + } else if (kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(backend_csf->enable_state)) { + /* We might already be in the TRANSITION_TO_DISABLED_* states and + * the MCU is handling outstanding ACKs due to MCU power cycle, + * so just ignore it. + */ } else { /* Unexpected state change, assume everything is broken until * we reset. @@ -2189,17 +2809,45 @@ void kbase_hwcnt_backend_csf_on_prfcnt_disable(struct kbase_hwcnt_backend_interf csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + /* We received a disable ACK, so we can decrement the ring buffer refcounter. */ + if (csf_info->ring_buf_refcnt.incremented_on_mcu_off) { + csf_info->ring_buf_refcnt.incremented_on_mcu_off = false; + kbasep_hwcnt_backend_csf_ring_buf_refcnt_decrement_nolock( + csf_info, NULL, KBASEP_RING_BUF_FREE_DEFERRED); + } + /* Early out if the backend does not exist. */ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) return; + backend_csf = csf_info->backend; - if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED) { - kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( - backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER); + if (backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK) { + /* If the disable REQ was deferred due to MCU being off, we have already waited + * for the workers to complete. Hence it's safe to transition directly to + * DISABLED state. + * + * In any other case + * (e.g. KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK + * was set due to an recoverable error), we need to wait for workers to be flushed + * at the next dump_disable() call. + */ + enum kbase_hwcnt_backend_csf_enable_state new_state = + backend_csf->flush_workers_needed ? + KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER : + KBASE_HWCNT_BACKEND_CSF_DISABLED; + backend_csf->flush_workers_needed = false; + + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters(backend_csf, new_state); + } else if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED) { - /* Unexpected, but we are already in the right state so just - * ignore it. + /* This might be expected if we are aligning the backend state with MCU state. + * We are already in the right state so just ignore it. + */ + } else if (kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(backend_csf->enable_state)) { + /* We might already be in the TRANSITION_TO_ENABLED_* states and the MCU is handling + * outstanding ACKs due to MCU power cycle, so just ignore it. */ } else { /* Unexpected state change, assume everything is broken until @@ -2209,6 +2857,272 @@ void kbase_hwcnt_backend_csf_on_prfcnt_disable(struct kbase_hwcnt_backend_interf } } +static void +kbasep_hwcnt_backend_csf_on_after_mcu_off_nolock(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); + + WARN_ON(!csf_info->mcu_on); + csf_info->mcu_on = false; + /* Early out if the backend does not exist. */ + if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) + return; + + backend_csf = csf_info->backend; + + /* Notify about the MCU OFF event. */ + wake_up(&backend_csf->enable_state_waitq); + + /* Ring buffer refcount makes sure the ring buffer is only freed when FW has ACKed + * any outstanding HWC disable REQs. This means that the refcount should be incremented + * on MCU_OFF event when the backend is enabled (or about to be enabled) from + * FW perspective, i.e. states: + * - KBASE_HWCNT_BACKEND_CSF_ENABLED, + * - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_*, because the disable + * is not yet ACKed, + * - KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK, to prevent + * a race condition between enable ACK IRQ and the ring buffer freeing + * after MCU ON notification. + * + * It's safe to free the ring buffer (i.e. to not increase the refcount) while MCU is OFF + * if the backend disable has been ACKed by FW or the enabling REQ never reached FW + * (e.g. KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ). + * + * Do not increase the refcnt again if FW still hasn't ACKed the backend disable since + * the last MCU power cycle. + */ + if (!backend_csf->info->ring_buf_refcnt.incremented_on_mcu_off && + (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED || + kbasep_hwcnt_backend_csf_is_state_trans_to_disabled(backend_csf->enable_state) || + backend_csf->enable_state == + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK)) { + kbasep_hwcnt_backend_csf_ring_buf_refcnt_increment_nolock(csf_info); + backend_csf->info->ring_buf_refcnt.incremented_on_mcu_off = true; + } + + kbasep_hwcnt_backend_csf_append_block_states(backend_csf, KBASE_HWCNT_STATE_OFF); + + /* ACK the sample */ + kbase_hwcnt_backend_csf_on_prfcnt_sample(iface); +} + +void kbase_hwcnt_backend_csf_on_after_mcu_off(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + unsigned long flags = 0UL; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + kbasep_hwcnt_backend_csf_on_after_mcu_off_nolock(iface); + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + +void kbase_hwcnt_backend_csf_on_after_mcu_off_reset(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + unsigned long flags = 0UL; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + + /* It's uncertain at which MCU state reset was called. + * Hence, only notify the backend about MCU_OFF if it hasn't been done yet. + */ + if (csf_info->mcu_on) + kbasep_hwcnt_backend_csf_on_after_mcu_off_nolock(iface); + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + +void kbase_hwcnt_backend_csf_on_after_mcu_on(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + const enum kbase_hwcnt_backend_csf_enable_state trans_to_disabled_pending_ack = + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK; + + unsigned long flags = 0UL; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + + WARN_ON(csf_info->mcu_on); + csf_info->mcu_on = true; + + if (kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + backend_csf = (struct kbase_hwcnt_backend_csf *)csf_info->backend; + + kbasep_hwcnt_backend_csf_append_block_states(backend_csf, KBASE_HWCNT_STATE_ON); + + switch (backend_csf->enable_state) { + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ: + /* pending_disable_req means that prior to enabling, we need to disable + * the backend first. + */ + if (backend_csf->info->pending_disable_req) { + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, trans_to_disabled_pending_ack); + /* Send the disable REQ now. */ + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); + + backend_csf->dump_reenable_in_progress = true; + /* Queue a worker to wait for disable ACK and to send enable REQ. */ + queue_work(backend_csf->info->hwc_deferred_op_workq, + &backend_csf->info->hwc_dump_reenable_work); + /* Exit the backend state handling. */ + break; + } + + /* At this point we are sure that the previous disable was ACKed by FW, + * so we can proceed directly to the enable REQ. + */ + if (kbasep_hwcnt_backend_csf_do_dump_enable(backend_csf, + &backend_csf->last_enable_map)) + kbasep_hwcnt_backend_csf_handle_unrecoverable_error(backend_csf); + break; + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_ACK: + case KBASE_HWCNT_BACKEND_CSF_ENABLED: + /* Resume the timer. */ + csf_info->watchdog_if->modify(csf_info->watchdog_if->timer, + csf_info->watchdog_timer_interval_ms); + break; + case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_REQ: + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED_PENDING_ACK); + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); + + break; + case KBASE_HWCNT_BACKEND_CSF_DISABLED: + /* Here we handle a case where the backend was recreated (but not yet + * desired to be enabled) while MCU was off. If the disable REQ + * never reached MCU, we need to apply it now to align the HWC state + * in MCU and in the kbase. + * The disable ACK should be ignored. + */ + if (backend_csf->info->pending_disable_req) + kbasep_hwcnt_backend_csf_send_disable_req(backend_csf->info); + + break; + default: + /* Do nothing. */ + break; + } + + /* Notify about the MCU ON event. */ + wake_up(&backend_csf->enable_state_waitq); + + } else { + /* Backend has been already destroyed. */ + if (csf_info->pending_disable_req) { + /* Finish the backend destruction by sending the disable REQ to FW. */ + kbasep_hwcnt_backend_csf_send_disable_req(csf_info); + } + } + + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + +void kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + struct kbase_hwcnt_backend_csf *backend_csf; + unsigned long flags = 0UL; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); + + WARN_ON(csf_info->mcu_on); + + /* Reset the internal variables to match the cold-booted FW state.*/ + csf_info->pending_disable_req = false; + + if (kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { + bool do_reenable = false; + + backend_csf = (struct kbase_hwcnt_backend_csf *)csf_info->backend; + + if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED || + kbasep_hwcnt_backend_csf_is_state_trans_to_enabled(backend_csf->enable_state)) { + /* If the backend was enabled (or was transitioning to) prior to the + * cold boot, we need to reenable it so it can continue to process + * dump requests after the boot. + */ + do_reenable = true; + + /* Disable watchdog in a non-blocking manner. Pending watchdog callback will + * safely early-exit when called. + * + * Exclude TRANSITIONING_TO_ENABLED_PENDING_REQ state because watchdog + * is disabled in this state. + */ + if (backend_csf->enable_state != + KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED_PENDING_REQ) + backend_csf->info->watchdog_if->disable( + backend_csf->info->watchdog_if->timer, + KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_NOT_BLOCK); + } + + backend_csf->flush_workers_needed = false; + /* Cancel any reenable work in progress. */ + backend_csf->dump_reenable_in_progress = false; + /* Set backend state to DISABLED and cancel any pending dump work + * by resetting dump-related variables. + * + * Do not leave an error state. A dump_disable and a subsequent reset is the only + * way to leave the error state. + */ + if (!kbasep_hwcnt_backend_csf_is_error_state(backend_csf->enable_state)) { + kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( + backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED); + backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; + complete_all(&backend_csf->dump_completed); + } + + if (WARN_ON(backend_csf->info->ring_buf == NULL)) + goto unlock; + + /* Keep using the allocated ring buffer. Reset its refcount to 1. */ + csf_info->ring_buf_refcnt = (struct kbasep_hwcnt_ring_buf_refcnt){ .val = 1 }; + /* Zero all performance enable header to prepare for first enable + * after a cold boot. + */ + kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header(backend_csf); + /* Sync zeroed buffers to avoid coherency issues on future use. */ + backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, + backend_csf->info->ring_buf, 0, + backend_csf->info->ring_buf_cnt, false); + + if (do_reenable) { + int errcode; + + errcode = kbasep_hwcnt_backend_csf_dump_enable_nolock( + (struct kbase_hwcnt_backend *)backend_csf, + &backend_csf->last_enable_map); + if (errcode) { + kbasep_hwcnt_backend_csf_handle_unrecoverable_error(backend_csf); + goto unlock; + } + } + } else { + csf_info->ring_buf_refcnt = (struct kbasep_hwcnt_ring_buf_refcnt){ 0 }; + if (csf_info->ring_buf) { + /* It's safe to ignore ring buffer's refcounter because HWC is disabled + * during the cold boot. + */ + kbasep_hwcnt_backend_csf_ring_buf_free_deferred(csf_info); + } + } + +unlock: + csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); +} + int kbase_hwcnt_backend_csf_metadata_init(struct kbase_hwcnt_backend_interface *iface) { struct kbase_hwcnt_backend_csf_info *csf_info; @@ -2268,6 +3182,29 @@ void kbase_hwcnt_backend_csf_metadata_term(struct kbase_hwcnt_backend_interface } } +void kbase_hwcnt_backend_csf_ring_buf_term(struct kbase_hwcnt_backend_interface *iface) +{ + struct kbase_hwcnt_backend_csf_info *csf_info; + + if (!iface) + return; + + csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; + + /* Wait for any ring_buf_free worker to finish. */ + flush_workqueue(csf_info->hwc_deferred_op_workq); + + /* Free ring buffer, if not yet freed. + * It's safe to ignore ring buffer's refcounter because firmware is being terminated, + * hence MCU will not be booted again. + * Needs to be done before firmware is terminated. + */ + if (csf_info->ring_buf != NULL) { + csf_info->csf_if->ring_buf_free(csf_info->csf_if->ctx, csf_info->ring_buf); + csf_info->ring_buf = NULL; + } +} + int kbase_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_if *csf_if, u32 ring_buf_cnt, struct kbase_hwcnt_watchdog_interface *watchdog_if, struct kbase_hwcnt_backend_interface *iface, diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h index 171b28deb6bc..5e3b96fa3d61 100644 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h +++ b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h @@ -31,10 +31,35 @@ #include "hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h" #include "hwcnt/mali_kbase_hwcnt_watchdog_if.h" #include "hwcnt/mali_kbase_hwcnt_types.h" +#include struct kbase_hwcnt_physical_enable_map; struct kbase_hwcnt_backend_csf; +/** + * enum kbase_hwcnt_backend_sample_reason - HWC CSF metadata block sample + * reasons. + * + * @SAMPLE_REASON_NONE: No metadata was enabled or available. + * @SAMPLE_REASON_BEFORE_PROTM: The sample was taken just before the GPU enters protected mode. + * @SAMPLE_REASON_BEFORE_HALT: The sample was taken just before the GPU enters the HALT state. + * @SAMPLE_REASON_BEFORE_SLEEP: The sample was taken just before the GPU enters the SLEEP state. + * @SAMPLE_REASON_BEFORE_SUSPEND: The sample was taken just before the GPU enters the SUSPEND + * state. + * @SAMPLE_REASON_BEFORE_YIELD: The sample was taken just before GPU subinstance yields the + * access window. + * @SAMPLE_REASON_AFTER_WARM_BOOT: The sample taken after GPU subinstance completed a warm boot. + */ +enum kbase_hwcnt_backend_sample_reason { + SAMPLE_REASON_NONE, + SAMPLE_REASON_BEFORE_PROTM, + SAMPLE_REASON_BEFORE_HALT, + SAMPLE_REASON_BEFORE_SLEEP, + SAMPLE_REASON_BEFORE_SUSPEND, + SAMPLE_REASON_BEFORE_YIELD, + SAMPLE_REASON_AFTER_WARM_BOOT +}; + /** * kbase_hwcnt_backend_csf_create() - Create a CSF hardware counter backend * interface, with set timer interval. @@ -72,6 +97,18 @@ int kbase_hwcnt_backend_csf_metadata_init(struct kbase_hwcnt_backend_interface * */ void kbase_hwcnt_backend_csf_metadata_term(struct kbase_hwcnt_backend_interface *iface); +/** + * kbase_hwcnt_backend_csf_ring_buf_term() - Terminate the ring buffers for a CSF + * hardware counter backend. Must be called while + * FW is still loaded. + * + * @iface: Non-NULL pointer to backend interface structure. + * + * If the ring buffer was not freed while HWC backend was present + * (e.g. due to MCU being powered off), this function will free it. + */ +void kbase_hwcnt_backend_csf_ring_buf_term(struct kbase_hwcnt_backend_interface *iface); + /** * kbase_hwcnt_backend_csf_destroy() - Destroy a CSF hardware counter backend * interface. @@ -123,8 +160,8 @@ void kbase_hwcnt_backend_csf_on_before_reset(struct kbase_hwcnt_backend_interfac /** * kbase_hwcnt_backend_csf_set_hw_availability() - CSF HWC backend function to * set current HW configuration. - * HWC must be disabled before - * this function is called. + * Can only be called when the MCU is off. + * * @iface: Non-NULL pointer to HWC backend interface. * @num_l2_slices: Current number of L2 slices allocated to the GPU. * @shader_present: Shader_present of the current configuration. @@ -179,21 +216,61 @@ void kbase_hwcnt_backend_csf_on_prfcnt_enable(struct kbase_hwcnt_backend_interfa */ void kbase_hwcnt_backend_csf_on_prfcnt_disable(struct kbase_hwcnt_backend_interface *iface); +/** + * kbase_hwcnt_backend_csf_on_after_mcu_off() - CSF HWC backend function to be called immediately + * after the MCU shut down process is completed, + * informing the backend that it is no longer valid + * to send commands to the MCU and that any + * outstanding commands will not be ACKed. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_after_mcu_off(struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_after_mcu_off_reset() - Similar to + * kbase_hwcnt_backend_csf_on_after_mcu_off(). + * To be called only after a completed reset. + * This function will notify backend about MCU + * being powered off only if it hasn't been + * done yet. This is due to the fact that it's + * uncertain if the MCU_OFF notification was + * raised prior to the reset. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_after_mcu_off_reset(struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_after_mcu_on() - CSF HWC backend function to be called immediately + * after the MCU has booted, informing the backend + * that it is now valid to send commands to the MCU. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_after_mcu_on(struct kbase_hwcnt_backend_interface *iface); + +/** + * kbase_hwcnt_backend_csf_on_before_mcu_cold_boot() - CSF HWC backend function to be called + * immediately before a cold MCU boot, + * allowing the backend to reset its internal + * state machine to match the cold-booted + * FW state. + * @iface: Non-NULL pointer to HWC backend interface. + */ +void kbase_hwcnt_backend_csf_on_before_mcu_cold_boot(struct kbase_hwcnt_backend_interface *iface); + /** * kbasep_hwcnt_backend_csf_update_block_state - Update block state of a block instance with * information from a sample. * @backend: CSF hardware counter backend. * @enable_mask: Counter enable mask for the block whose state is being updated. - * @exiting_protm: Whether or not the sample is taken when the GPU is exiting - * protected mode. - * @block_idx: Index of block within the ringbuffer. + * @block_idx: Index of the block in the dump. * @block_state: Pointer to existing block state of the block whose state is being * updated. - * @fw_in_protected_mode: Whether or not GPU is in protected mode during sampling. + * @prev_sample_reason: The previous sample reason + * @curr_sample_reason: The current sample reason */ -void kbasep_hwcnt_backend_csf_update_block_state(struct kbase_hwcnt_backend_csf *backend, - const u32 enable_mask, bool exiting_protm, - size_t block_idx, blk_stt_t *const block_state, - bool fw_in_protected_mode); +void kbasep_hwcnt_backend_csf_update_block_state( + struct kbase_hwcnt_backend_csf *backend, const u32 enable_mask, size_t block_idx, + blk_stt_t *const block_state, enum kbase_hwcnt_backend_sample_reason prev_sample_reason, + enum kbase_hwcnt_backend_sample_reason curr_sample_reason); #endif /* _KBASE_HWCNT_BACKEND_CSF_H_ */ diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h index 220a63de779a..f8ba2acb4b67 100644 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h +++ b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h @@ -61,15 +61,16 @@ struct kbase_hwcnt_backend_csf_if_enable { /** * struct kbase_hwcnt_backend_csf_if_prfcnt_info - Performance counter * information. - * @prfcnt_hw_size: Total length in bytes of all the hardware counters data. The hardware - * counters are sub-divided into 4 classes: front-end, shader, tiler, and - * memory system (l2 cache + MMU). - * @prfcnt_fw_size: Total length in bytes of all the firmware counters data. - * @dump_bytes: Bytes of GPU memory required to perform a performance - * counter dump. dump_bytes = prfcnt_hw_size + prfcnt_fw_size. - * @prfcnt_block_size: Bytes of each performance counter block. - * @l2_count: The MMU L2 cache count. - * @csg_count: The total number of CSGs in the system + * @prfcnt_hw_size: Total length in bytes of all the hardware counters data. The hardware + * counters are sub-divided into 4 classes: front-end, shader, tiler, and + * memory system (l2 cache + MMU). + * @prfcnt_fw_size: Total length in bytes of all the firmware counters data. + * @metadata_size: Total length in bytes of the dedicated metadata block. + * @dump_bytes: Bytes of GPU memory required to perform a performance counter dump. + * dump_bytes = prfcnt_metadata_size + prfcnt_hw_size + prfcnt_fw_size. + * @prfcnt_block_size: Bytes of each performance counter block. + * @l2_count: The MMU L2 cache count. + * @csg_count: The total number of CSGs in the system * @sc_core_mask: Shader core mask. * @clk_cnt: Clock domain count in the system. * @clearing_samples: Indicates whether counters are cleared after each sample @@ -81,6 +82,7 @@ struct kbase_hwcnt_backend_csf_if_enable { struct kbase_hwcnt_backend_csf_if_prfcnt_info { size_t prfcnt_hw_size; size_t prfcnt_fw_size; + size_t metadata_size; size_t dump_bytes; size_t prfcnt_block_size; size_t l2_count; diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c index 15f5ff4987f0..93d6f0d656d4 100644 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c +++ b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c @@ -303,6 +303,7 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( KBASE_DUMMY_MODEL_MAX_NUM_HARDWARE_BLOCKS * KBASE_DUMMY_MODEL_BLOCK_SIZE, .prfcnt_fw_size = KBASE_DUMMY_MODEL_MAX_FIRMWARE_BLOCKS * KBASE_DUMMY_MODEL_BLOCK_SIZE, + .metadata_size = 0, .dump_bytes = KBASE_DUMMY_MODEL_MAX_SAMPLE_SIZE, .prfcnt_block_size = KBASE_DUMMY_MODEL_BLOCK_SIZE, .clk_cnt = 1, @@ -317,6 +318,8 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( u32 prfcnt_size; u32 prfcnt_hw_size; u32 prfcnt_fw_size; + u32 prfcnt_features; + u32 metadata_size; u32 csg_count; u32 fw_block_count = 0; u32 prfcnt_block_size = @@ -330,8 +333,10 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( kbdev = fw_ctx->kbdev; csg_count = kbdev->csf.global_iface.group_num; prfcnt_size = kbdev->csf.global_iface.prfcnt_size; + prfcnt_features = kbdev->csf.global_iface.prfcnt_features; prfcnt_hw_size = GLB_PRFCNT_SIZE_HARDWARE_SIZE_GET(prfcnt_size); prfcnt_fw_size = GLB_PRFCNT_SIZE_FIRMWARE_SIZE_GET(prfcnt_size); + metadata_size = GLB_PRFCNT_FEATURES_METADATA_SIZE_GET(prfcnt_features); has_virtual_core_ids = kbdev->gpu_props.gpu_id.arch_id >= GPU_ID_ARCH_MAKE(14, 8, 4); /* Read the block size if the GPU has the register PRFCNT_FEATURES @@ -355,10 +360,11 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( else WARN_ON_ONCE(true); - fw_ctx->buf_bytes = prfcnt_hw_size + prfcnt_fw_size; + fw_ctx->buf_bytes = prfcnt_hw_size + prfcnt_fw_size + metadata_size; *prfcnt_info = (struct kbase_hwcnt_backend_csf_if_prfcnt_info){ .prfcnt_hw_size = prfcnt_hw_size, .prfcnt_fw_size = prfcnt_fw_size, + .metadata_size = metadata_size, .dump_bytes = fw_ctx->buf_bytes, .prfcnt_block_size = prfcnt_block_size, .l2_count = kbdev->gpu_props.num_l2_slices, @@ -380,10 +386,17 @@ static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( else WARN_ON(prfcnt_info->ne_core_mask != 0); + WARN((metadata_size % prfcnt_info->prfcnt_block_size) != 0, + "Metadata block size is not aligned to block size (metadata size %u, block size %zu)", + metadata_size, prfcnt_info->prfcnt_block_size); /* Block size must be multiple of counter size. */ - WARN_ON((prfcnt_info->prfcnt_block_size % KBASE_HWCNT_VALUE_HW_BYTES) != 0); + WARN((prfcnt_info->prfcnt_block_size % KBASE_HWCNT_VALUE_HW_BYTES) != 0, + "Block size is not a multiple of counter size (block size %zu, counter size %lu)", + prfcnt_info->prfcnt_block_size, (unsigned long)KBASE_HWCNT_VALUE_HW_BYTES); /* Total size must be multiple of block size. */ - WARN_ON((prfcnt_info->dump_bytes % prfcnt_info->prfcnt_block_size) != 0); + WARN((prfcnt_info->dump_bytes % prfcnt_info->prfcnt_block_size) != 0, + "Total size should be a multiple of block size (total size %zu, block size %zu)", + prfcnt_info->dump_bytes, prfcnt_info->prfcnt_block_size); #endif } @@ -440,8 +453,7 @@ static int kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc( goto page_list_alloc_error; /* Get physical page for the buffer */ - ret = kbase_mem_pool_alloc_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, - phys, false, NULL); + ret = kbase_mem_pool_alloc_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, NULL); if ((size_t)ret != num_pages) goto phys_mem_pool_alloc_error; @@ -484,8 +496,7 @@ static int kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc( mmu_insert_failed: vunmap(cpu_addr); vmap_error: - kbase_mem_pool_free_pages(&kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, phys, - false, false); + kbase_mem_pool_free_pages(&kbdev->fw_mem_pools.small, num_pages, phys, false, false); phys_mem_pool_alloc_error: kfree(page_list); page_list_alloc_error: @@ -604,7 +615,7 @@ kbasep_hwcnt_backend_csf_if_fw_ring_buf_free(struct kbase_hwcnt_backend_csf_if_c /* After zeroing, the ring_buf pages are dirty so need to pass the 'dirty' flag * as true when freeing the pages to the Global pool. */ - kbase_mem_pool_free_pages(&fw_ctx->kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], + kbase_mem_pool_free_pages(&fw_ctx->kbdev->fw_mem_pools.small, fw_ring_buf->num_pages, fw_ring_buf->phys, true, false); kfree(fw_ring_buf->phys); @@ -640,6 +651,7 @@ kbasep_hwcnt_backend_csf_if_fw_dump_enable(struct kbase_hwcnt_backend_csf_if_ctx /* Configure */ prfcnt_config = GLB_PRFCNT_CONFIG_SIZE_SET(0, fw_ring_buf->buf_count); prfcnt_config = GLB_PRFCNT_CONFIG_SET_SELECT_SET(prfcnt_config, enable->counter_set); + prfcnt_config = GLB_PRFCNT_CONFIG_METADATA_ENABLE_SET(prfcnt_config, 1); /* Configure the ring buffer base address */ kbase_csf_fw_io_global_write(&kbdev->csf.fw_io, GLB_PRFCNT_JASID, fw_ring_buf->as_nr); @@ -808,15 +820,27 @@ kbasep_hwcnt_backend_csf_if_fw_get_gpu_cycle_count(struct kbase_hwcnt_backend_cs if (!(clk_enable_map & (1ull << clk))) continue; - if (clk == KBASE_CLOCK_DOMAIN_TOP) { + /* Collect cycle count from GPU register. + * If GPU is unavailable, fall back to a SW estimation. + */ + if (clk == KBASE_CLOCK_DOMAIN_TOP && kbase_io_has_gpu(fw_ctx->kbdev)) { /* Read cycle count for top clock domain. */ kbase_backend_get_gpu_time_norequest(fw_ctx->kbdev, &cycle_counts[clk], NULL, NULL); - } else { - /* Estimate cycle count for non-top clock domain. */ - cycle_counts[clk] = - kbase_ccswe_cycle_at(&fw_ctx->ccswe_shader_cores, timestamp_ns); + + /* Check again if the device is lost. + * If we still have GPU, then the value is correct and we can continue with + * the next loop iteration. + * If it's lost, we can't guarantee we have read good data out + * from CYCLE_COUNT, so fall back to the SW estimated cycle count. + */ + if (kbase_io_has_gpu(fw_ctx->kbdev)) + continue; } + /* Estimate cycle count for non-top clock domain or for the top clock domain while + * the device is lost. + */ + cycle_counts[clk] = kbase_ccswe_cycle_at(&fw_ctx->ccswe_shader_cores, timestamp_ns); } } diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c deleted file mode 100644 index 3467c5fd4d8b..000000000000 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c +++ /dev/null @@ -1,981 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include "hwcnt/backend/mali_kbase_hwcnt_backend_jm.h" -#include "hwcnt/mali_kbase_hwcnt_gpu.h" -#include "hwcnt/mali_kbase_hwcnt_types.h" -#include "mali_kbase.h" -#include "mali_kbase_mem_flags.h" -#include "backend/gpu/mali_kbase_pm_ca.h" -#include "mali_kbase_hwaccess_instr.h" -#include "mali_kbase_hwaccess_time.h" -#include "mali_kbase_ccswe.h" -#include "backend/gpu/mali_kbase_model_linux.h" -#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" - -#include "backend/gpu/mali_kbase_pm_internal.h" - -/** - * struct kbase_hwcnt_backend_jm_info - Information used to create an instance - * of a JM hardware counter backend. - * @kbdev: KBase device. - * @counter_set: The performance counter set to use. - * @metadata: Hardware counter metadata. - * @dump_bytes: Bytes of GPU memory required to perform a - * hardware counter dump. - * @hwcnt_gpu_info: Hardware counter block information. - */ -struct kbase_hwcnt_backend_jm_info { - struct kbase_device *kbdev; - enum kbase_hwcnt_set counter_set; - const struct kbase_hwcnt_metadata *metadata; - size_t dump_bytes; - struct kbase_hwcnt_gpu_info hwcnt_gpu_info; -}; - -/** - * struct kbase_hwcnt_jm_physical_layout - HWC sample memory physical layout - * information. - * @fe_cnt: Front end block count. - * @tiler_cnt: Tiler block count. - * @mmu_l2_cnt: Memory system(MMU and L2 cache) block count. - * @shader_cnt: Shader Core block count. - * @block_cnt: Total block count (sum of all other block counts). - * @shader_avail_mask: Bitmap of all shader cores in the system. - * @enable_mask_offset: Offset in array elements of enable mask in each block - * starting from the beginning of block. - * @headers_per_block: Header size per block. - * @counters_per_block: Counters size per block. - * @values_per_block: Total size per block. - */ -struct kbase_hwcnt_jm_physical_layout { - u8 fe_cnt; - u8 tiler_cnt; - u8 mmu_l2_cnt; - u8 shader_cnt; - u8 block_cnt; - u64 shader_avail_mask; - size_t enable_mask_offset; - size_t headers_per_block; - size_t counters_per_block; - size_t values_per_block; -}; - -/** - * struct kbase_hwcnt_backend_jm - Instance of a JM hardware counter backend. - * @info: Info used to create the backend. - * @kctx: KBase context used for GPU memory allocation and - * counter dumping. - * @gpu_dump_va: GPU hardware counter dump buffer virtual address. - * @cpu_dump_va: CPU mapping of gpu_dump_va. - * @vmap: Dump buffer vmap. - * @to_user_buf: HWC sample buffer for client user, size - * metadata.dump_buf_bytes. - * @to_user_block_states: HWC sample block states for client user. - * @enabled: True if dumping has been enabled, else false. - * @debug_core_mask: User-set mask of shader cores that can be used. - * @pm_core_mask: PM state sync-ed shaders core mask for the enabled - * dumping. - * @curr_config: Current allocated hardware resources to correctly map the - * source raw dump buffer to the destination dump buffer. - * @max_core_mask: Core mask of all cores allocated to the GPU (non - * virtualized platforms) or resource group (virtualized - * platforms). - * @max_l2_slices: Maximum number of L2 slices allocated to the GPU (non - * virtualized platforms) or resource group (virtualized - * platforms). - * @clk_enable_map: The enable map specifying enabled clock domains. - * @cycle_count_elapsed: Cycle count elapsed for a given sample period. - * The top clock cycle, index 0, is read directly from - * hardware, but the other clock domains need to be - * calculated with software estimation. - * @prev_cycle_count: Previous cycle count to calculate the cycle count for - * sample period. - * @rate_listener: Clock rate listener callback state. - * @ccswe_shader_cores: Shader cores cycle count software estimator. - * @phys_layout: Physical memory layout information of HWC sample buffer. - * @dump_time_ns: Holds the CPU timestamp captured at the time of - * dump_request(). - */ -struct kbase_hwcnt_backend_jm { - const struct kbase_hwcnt_backend_jm_info *info; - struct kbase_context *kctx; - u64 gpu_dump_va; - void *cpu_dump_va; - struct kbase_vmap_struct *vmap; - u64 *to_user_buf; - blk_stt_t *to_user_block_states; - bool enabled; - u64 debug_core_mask; - u64 pm_core_mask; - struct kbase_hwcnt_curr_config curr_config; - u64 max_core_mask; - size_t max_l2_slices; - u64 clk_enable_map; - u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS]; - u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS]; - struct kbase_clk_rate_listener rate_listener; - struct kbase_ccswe ccswe_shader_cores; - struct kbase_hwcnt_jm_physical_layout phys_layout; - u64 dump_time_ns; -}; - -/** - * kbasep_hwcnt_backend_jm_gpu_info_init() - Initialise an info structure used - * to create the hwcnt metadata. - * @kbdev: Non-NULL pointer to kbase device. - * @info: Non-NULL pointer to data structure to be filled in. - * - * The initialised info struct will only be valid for use while kbdev is valid. - * - * Return: 0 on success, else error code. - */ -static int kbasep_hwcnt_backend_jm_gpu_info_init(struct kbase_device *kbdev, - struct kbase_hwcnt_gpu_info *info) -{ - size_t clk, l2_count, core_mask; - - if (!kbdev || !info) - return -EINVAL; - -#if IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) - l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS; - core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1; -#else - l2_count = kbdev->gpu_props.num_l2_slices; - core_mask = kbdev->gpu_props.coherency_info.group.core_mask; -#endif - - info->l2_count = l2_count; - info->sc_core_mask = core_mask; - info->prfcnt_values_per_block = KBASE_HWCNT_V5_DEFAULT_VALUES_PER_BLOCK; - - /* Determine the number of available clock domains. */ - for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) { - if (kbdev->pm.clk_rtm.clks[clk] == NULL) - break; - } - info->clk_cnt = clk; - - return 0; -} - -static void kbasep_hwcnt_backend_jm_init_layout(const struct kbase_hwcnt_gpu_info *gpu_info, - struct kbase_hwcnt_jm_physical_layout *phys_layout) -{ - u8 shader_core_cnt; - - WARN_ON(!gpu_info); - WARN_ON(!phys_layout); - - shader_core_cnt = fls64(gpu_info->sc_core_mask); - - *phys_layout = (struct kbase_hwcnt_jm_physical_layout){ - .fe_cnt = KBASE_HWCNT_V5_FE_BLOCK_COUNT, - .tiler_cnt = KBASE_HWCNT_V5_TILER_BLOCK_COUNT, - .mmu_l2_cnt = gpu_info->l2_count, - .shader_cnt = shader_core_cnt, - .block_cnt = KBASE_HWCNT_V5_FE_BLOCK_COUNT + KBASE_HWCNT_V5_TILER_BLOCK_COUNT + - gpu_info->l2_count + shader_core_cnt, - .shader_avail_mask = gpu_info->sc_core_mask, - .headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK, - .values_per_block = gpu_info->prfcnt_values_per_block, - .counters_per_block = - gpu_info->prfcnt_values_per_block - KBASE_HWCNT_V5_HEADERS_PER_BLOCK, - .enable_mask_offset = KBASE_HWCNT_V5_PRFCNT_EN_HEADER, - }; -} - -static void -kbasep_hwcnt_backend_jm_dump_sample(const struct kbase_hwcnt_backend_jm *const backend_jm) -{ - size_t block_idx; - const u32 *new_sample_buf = backend_jm->cpu_dump_va; - const u32 *new_block = new_sample_buf; - u64 *dst_buf = backend_jm->to_user_buf; - u64 *dst_block = dst_buf; - const size_t values_per_block = backend_jm->phys_layout.values_per_block; - const size_t dump_bytes = backend_jm->info->dump_bytes; - - for (block_idx = 0; block_idx < backend_jm->phys_layout.block_cnt; block_idx++) { - size_t ctr_idx; - - for (ctr_idx = 0; ctr_idx < values_per_block; ctr_idx++) - dst_block[ctr_idx] = new_block[ctr_idx]; - - new_block += values_per_block; - dst_block += values_per_block; - } - - WARN_ON(new_block != new_sample_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES)); - WARN_ON(dst_block != dst_buf + (dump_bytes / KBASE_HWCNT_VALUE_HW_BYTES)); -} - -/** - * kbasep_hwcnt_backend_jm_on_freq_change() - On freq change callback - * - * @rate_listener: Callback state - * @clk_index: Clock index - * @clk_rate_hz: Clock frequency(hz) - */ -static void kbasep_hwcnt_backend_jm_on_freq_change(struct kbase_clk_rate_listener *rate_listener, - u32 clk_index, u32 clk_rate_hz) -{ - struct kbase_hwcnt_backend_jm *backend_jm = - container_of(rate_listener, struct kbase_hwcnt_backend_jm, rate_listener); - u64 timestamp_ns; - - if (clk_index != KBASE_CLOCK_DOMAIN_SHADER_CORES) - return; - - timestamp_ns = ktime_get_raw_ns(); - kbase_ccswe_freq_change(&backend_jm->ccswe_shader_cores, timestamp_ns, clk_rate_hz); -} - -/** - * kbasep_hwcnt_backend_jm_cc_enable() - Enable cycle count tracking - * - * @backend_jm: Non-NULL pointer to backend. - * @enable_map: Non-NULL pointer to enable map specifying enabled counters. - * @timestamp_ns: Timestamp(ns) when HWCNT were enabled. - */ -static void kbasep_hwcnt_backend_jm_cc_enable(struct kbase_hwcnt_backend_jm *backend_jm, - const struct kbase_hwcnt_enable_map *enable_map, - u64 timestamp_ns) -{ - struct kbase_device *kbdev = backend_jm->kctx->kbdev; - u64 clk_enable_map = enable_map->clk_enable_map; - u64 cycle_count; - - if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { - /* turn on the cycle counter */ - kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev); - /* Read cycle count for top clock domain. */ - kbase_backend_get_gpu_time_norequest(kbdev, &cycle_count, NULL, NULL); - - backend_jm->prev_cycle_count[KBASE_CLOCK_DOMAIN_TOP] = cycle_count; - } - - if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { - /* software estimation for non-top clock domains */ - struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; - const struct kbase_clk_data *clk_data = rtm->clks[KBASE_CLOCK_DOMAIN_SHADER_CORES]; - u32 cur_freq; - unsigned long flags; - - spin_lock_irqsave(&rtm->lock, flags); - - cur_freq = (u32)clk_data->clock_val; - kbase_ccswe_reset(&backend_jm->ccswe_shader_cores); - kbase_ccswe_freq_change(&backend_jm->ccswe_shader_cores, timestamp_ns, cur_freq); - - kbase_clk_rate_trace_manager_subscribe_no_lock(rtm, &backend_jm->rate_listener); - - spin_unlock_irqrestore(&rtm->lock, flags); - - /* ccswe was reset. The estimated cycle is zero. */ - backend_jm->prev_cycle_count[KBASE_CLOCK_DOMAIN_SHADER_CORES] = 0; - } - - /* Keep clk_enable_map for dump_request. */ - backend_jm->clk_enable_map = clk_enable_map; -} - -/** - * kbasep_hwcnt_backend_jm_cc_disable() - Disable cycle count tracking - * - * @backend_jm: Non-NULL pointer to backend. - */ -static void kbasep_hwcnt_backend_jm_cc_disable(struct kbase_hwcnt_backend_jm *backend_jm) -{ - struct kbase_device *kbdev = backend_jm->kctx->kbdev; - struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; - u64 clk_enable_map = backend_jm->clk_enable_map; - - if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { - /* turn off the cycle counter */ - kbase_pm_release_gpu_cycle_counter(kbdev); - } - - if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { - kbase_clk_rate_trace_manager_unsubscribe(rtm, &backend_jm->rate_listener); - } -} - -/** - * kbasep_hwcnt_gpu_update_curr_config() - Update the destination buffer with - * current config information. - * @kbdev: Non-NULL pointer to kbase device. - * @curr_config: Non-NULL pointer to return the current configuration of - * hardware allocated to the GPU. - * - * The current configuration information is used for architectures where the - * max_config interface is available from the Arbiter. In this case the current - * allocated hardware is not always the same, so the current config information - * is used to correctly map the current allocated resources to the memory layout - * that is copied to the user space. - * - * Return: 0 on success, else error code. - */ -static int kbasep_hwcnt_gpu_update_curr_config(struct kbase_device *kbdev, - struct kbase_hwcnt_curr_config *curr_config) -{ - if (WARN_ON(!kbdev) || WARN_ON(!curr_config)) - return -EINVAL; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - curr_config->num_l2_slices = kbdev->gpu_props.curr_config.l2_slices; - curr_config->shader_present = kbdev->gpu_props.curr_config.shader_present; - return 0; -} - -/* JM backend implementation of kbase_hwcnt_backend_timestamp_ns_fn */ -static u64 kbasep_hwcnt_backend_jm_timestamp_ns(struct kbase_hwcnt_backend *backend) -{ - (void)backend; - return ktime_get_raw_ns(); -} - -static void kbasep_hwcnt_backend_jm_append_block_states(struct kbase_hwcnt_backend_jm *backend_jm, - blk_stt_t block_state) -{ - size_t i; - - for (i = 0; i < backend_jm->phys_layout.block_cnt; i++) - kbase_hwcnt_block_state_append(&backend_jm->to_user_block_states[i], block_state); -} - -static void -kbasep_hwcnt_backend_jm_get_physical_enable(struct kbase_hwcnt_backend_jm *backend_jm, - const struct kbase_hwcnt_enable_map *enable_map, - struct kbase_instr_hwcnt_enable *enable) -{ - enum kbase_hwcnt_physical_set phys_counter_set; - struct kbase_hwcnt_physical_enable_map phys_enable_map = { 0 }; - - kbase_hwcnt_gpu_enable_map_to_physical(&phys_enable_map, enable_map); - - kbase_hwcnt_gpu_set_to_physical(&phys_counter_set, backend_jm->info->counter_set); - - *enable = (struct kbase_instr_hwcnt_enable){ - .fe_bm = phys_enable_map.fe_bm, - .shader_bm = phys_enable_map.shader_bm, - .tiler_bm = phys_enable_map.tiler_bm, - .mmu_l2_bm = phys_enable_map.mmu_l2_bm, - .counter_set = phys_counter_set, - .dump_buffer = backend_jm->gpu_dump_va, - .dump_buffer_bytes = backend_jm->info->dump_bytes, - }; - - /* The dummy model needs the CPU mapping. */ - if (IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI)) - enable->dump_buffer = (uintptr_t)backend_jm->cpu_dump_va; -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ -static int -kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend *backend, - const struct kbase_hwcnt_enable_map *enable_map) -{ - int errcode; - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - struct kbase_context *kctx; - struct kbase_device *kbdev; - struct kbase_instr_hwcnt_enable enable = { 0 }; - u64 timestamp_ns; - - if (!backend_jm || !enable_map || backend_jm->enabled || - (enable_map->metadata != backend_jm->info->metadata)) - return -EINVAL; - - kctx = backend_jm->kctx; - kbdev = backend_jm->kctx->kbdev; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - kbasep_hwcnt_backend_jm_get_physical_enable(backend_jm, enable_map, &enable); - - timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); - - /* Update the current configuration information. */ - errcode = kbasep_hwcnt_gpu_update_curr_config(kbdev, &backend_jm->curr_config); - if (errcode) - goto error; - - errcode = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &enable); - if (errcode) - goto error; - - backend_jm->debug_core_mask = kbase_pm_ca_get_debug_core_mask(kbdev); - backend_jm->max_l2_slices = backend_jm->info->hwcnt_gpu_info.l2_count; - backend_jm->max_core_mask = backend_jm->info->hwcnt_gpu_info.sc_core_mask; - - backend_jm->pm_core_mask = kbase_pm_ca_get_instr_core_mask(kbdev); - - backend_jm->enabled = true; - - /* Enabling counters is an indication that the power may have previously been off for all - * blocks. - * - * In any case, the counters would not have been counting recently, so an 'off' block state - * is an approximation for this. - * - * This will be transferred to the dump only after a dump_wait(), or dump_disable() in - * cases where the caller requested such information. This is to handle when a - * dump_enable() happens in between dump_wait() and dump_get(). - */ - kbasep_hwcnt_backend_jm_append_block_states(backend_jm, KBASE_HWCNT_STATE_OFF); - - kbasep_hwcnt_backend_jm_cc_enable(backend_jm, enable_map, timestamp_ns); - - return 0; -error: - return errcode; -} - -static void -kbasep_hwcnt_backend_jm_reset_consumed_buffers(struct kbase_hwcnt_backend_jm *backend_jm) -{ - size_t block_state_bytes = backend_jm->phys_layout.block_cnt * - KBASE_HWCNT_BLOCK_STATE_BYTES * KBASE_HWCNT_BLOCK_STATE_STRIDE; - - memset(backend_jm->to_user_block_states, 0, block_state_bytes); -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_enable_fn */ -static int kbasep_hwcnt_backend_jm_dump_enable(struct kbase_hwcnt_backend *backend, - const struct kbase_hwcnt_enable_map *enable_map) -{ - unsigned long flags; - int errcode; - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - struct kbase_device *kbdev; - - if (!backend_jm) - return -EINVAL; - - kbdev = backend_jm->kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - errcode = kbasep_hwcnt_backend_jm_dump_enable_nolock(backend, enable_map); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return errcode; -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_disable_fn */ -static void kbasep_hwcnt_backend_jm_dump_disable(struct kbase_hwcnt_backend *backend, - struct kbase_hwcnt_dump_buffer *dump_buffer, - const struct kbase_hwcnt_enable_map *enable_map) -{ - int errcode; - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - - if (WARN_ON(!backend_jm || - (dump_buffer && (backend_jm->info->metadata != dump_buffer->metadata)) || - (enable_map && (backend_jm->info->metadata != enable_map->metadata)) || - (dump_buffer && !enable_map))) - return; - - /* No WARN needed here, but still return early if backend is already disabled */ - if (!backend_jm->enabled) - return; - - kbasep_hwcnt_backend_jm_cc_disable(backend_jm); - - errcode = kbase_instr_hwcnt_disable_internal(backend_jm->kctx); - WARN_ON(errcode); - - kbasep_hwcnt_backend_jm_append_block_states(backend_jm, KBASE_HWCNT_STATE_OFF); - - if (dump_buffer) { - /* In some use-cases, the caller will need the information whilst the counters are - * disabled, but will not be able to call into the backend to dump them. Instead, - * they have an opportunity here to request them to be accumulated into their - * buffer immediately. - * - * This consists of taking a sample of the accumulated block state (as though a - * real dump_get() had happened), then transfer ownership of that to the caller - * (i.e. erasing our copy of it). - */ - kbase_hwcnt_dump_buffer_append_block_states(dump_buffer, enable_map, - backend_jm->to_user_block_states); - - /* Now the block state has been passed out into the caller's own accumulation - * buffer, clear our own accumulated and sampled block state - ownership has been - * transferred. - */ - kbasep_hwcnt_backend_jm_reset_consumed_buffers(backend_jm); - } - - backend_jm->enabled = false; -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_clear_fn */ -static int kbasep_hwcnt_backend_jm_dump_clear(struct kbase_hwcnt_backend *backend) -{ - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - - if (!backend_jm || !backend_jm->enabled) - return -EINVAL; - - return kbase_instr_hwcnt_clear(backend_jm->kctx); -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_request_fn */ -static int kbasep_hwcnt_backend_jm_dump_request(struct kbase_hwcnt_backend *backend) -{ - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - struct kbase_device *kbdev; - const struct kbase_hwcnt_metadata *metadata; - u64 current_cycle_count; - size_t clk; - int ret; - - if (!backend_jm || !backend_jm->enabled) - return -EINVAL; - - kbdev = backend_jm->kctx->kbdev; - metadata = backend_jm->info->metadata; - - /* Disable pre-emption, to make the timestamp as accurate as possible */ - preempt_disable(); - { - backend_jm->dump_time_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); - ret = kbase_instr_hwcnt_request_dump(backend_jm->kctx); - - kbase_hwcnt_metadata_for_each_clock(metadata, clk) { - if (!kbase_hwcnt_clk_enable_map_enabled(backend_jm->clk_enable_map, clk)) - continue; - - if (clk == KBASE_CLOCK_DOMAIN_TOP) { - /* Read cycle count for top clock domain. */ - kbase_backend_get_gpu_time_norequest(kbdev, ¤t_cycle_count, - NULL, NULL); - } else { - /* - * Estimate cycle count for non-top clock - * domain. - */ - current_cycle_count = kbase_ccswe_cycle_at( - &backend_jm->ccswe_shader_cores, backend_jm->dump_time_ns); - } - backend_jm->cycle_count_elapsed[clk] = - current_cycle_count - backend_jm->prev_cycle_count[clk]; - - /* - * Keep the current cycle count for later calculation. - */ - backend_jm->prev_cycle_count[clk] = current_cycle_count; - } - } - preempt_enable(); - - return ret; -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_wait_fn */ -static int kbasep_hwcnt_backend_jm_dump_wait(struct kbase_hwcnt_backend *backend) -{ - int errcode; - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - - if (!backend_jm || !backend_jm->enabled) - return -EINVAL; - - errcode = kbase_instr_hwcnt_wait_for_dump(backend_jm->kctx); - if (errcode) - return errcode; - - return errcode; -} - -/* JM backend implementation of kbase_hwcnt_backend_dump_get_fn */ -static int kbasep_hwcnt_backend_jm_dump_get(struct kbase_hwcnt_backend *backend, - struct kbase_hwcnt_dump_buffer *dst, - const struct kbase_hwcnt_enable_map *dst_enable_map, - bool accumulate, u64 *dump_time_ns) -{ - struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; - size_t clk; -#if IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) - struct kbase_device *kbdev; - unsigned long flags; -#endif /* CONFIG_MALI_VALHALL_NO_MALI */ - int errcode; - - if (!backend_jm || !dst || !dst_enable_map || !dump_time_ns || - (backend_jm->info->metadata != dst->metadata) || - (dst_enable_map->metadata != dst->metadata)) - return -EINVAL; - - /* Invalidate the kernel buffer before reading from it. */ - kbase_sync_mem_regions(backend_jm->kctx, backend_jm->vmap, KBASE_SYNC_TO_CPU); - - /* Dump sample to the internal 64-bit user buffer. */ - kbasep_hwcnt_backend_jm_dump_sample(backend_jm); - - /* Extract elapsed cycle count for each clock domain if enabled. */ - kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) { - if (!kbase_hwcnt_clk_enable_map_enabled(dst_enable_map->clk_enable_map, clk)) - continue; - - /* Reset the counter to zero if accumulation is off. */ - if (!accumulate) - dst->clk_cnt_buf[clk] = 0; - dst->clk_cnt_buf[clk] += backend_jm->cycle_count_elapsed[clk]; - } - -#if IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) - kbdev = backend_jm->kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* Update the current configuration information. */ - errcode = kbasep_hwcnt_gpu_update_curr_config(kbdev, &backend_jm->curr_config); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (errcode) - return errcode; -#endif /* CONFIG_MALI_VALHALL_NO_MALI */ - errcode = kbase_hwcnt_jm_dump_get(dst, backend_jm->to_user_buf, dst_enable_map, - backend_jm->pm_core_mask, backend_jm->debug_core_mask, - backend_jm->max_l2_slices, &backend_jm->curr_config, - accumulate); - - if (errcode) - return errcode; - - *dump_time_ns = backend_jm->dump_time_ns; - kbase_hwcnt_dump_buffer_append_block_states(dst, dst_enable_map, - backend_jm->to_user_block_states); - kbasep_hwcnt_backend_jm_reset_consumed_buffers(backend_jm); - - return errcode; -} - -/** - * kbasep_hwcnt_backend_jm_dump_alloc() - Allocate a GPU dump buffer. - * @info: Non-NULL pointer to JM backend info. - * @kctx: Non-NULL pointer to kbase context. - * @gpu_dump_va: Non-NULL pointer to where GPU dump buffer virtual address - * is stored on success. - * - * Return: 0 on success, else error code. - */ -static int kbasep_hwcnt_backend_jm_dump_alloc(const struct kbase_hwcnt_backend_jm_info *info, - struct kbase_context *kctx, u64 *gpu_dump_va) -{ - struct kbase_va_region *reg; - base_mem_alloc_flags flags; - u64 nr_pages; - - /* Calls to this function are inherently asynchronous, with respect to - * MMU operations. - */ - const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC; - - WARN_ON(!info); - WARN_ON(!kctx); - WARN_ON(!gpu_dump_va); - - flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR | BASEP_MEM_PERMANENT_KERNEL_MAPPING | - BASE_MEM_CACHED_CPU | BASE_MEM_UNCACHED_GPU; - - nr_pages = PFN_UP(info->dump_bytes); - - reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, gpu_dump_va, mmu_sync_info); - - if (!reg) - return -ENOMEM; - - return 0; -} - -/** - * kbasep_hwcnt_backend_jm_dump_free() - Free an allocated GPU dump buffer. - * @kctx: Non-NULL pointer to kbase context. - * @gpu_dump_va: GPU dump buffer virtual address. - */ -static void kbasep_hwcnt_backend_jm_dump_free(struct kbase_context *kctx, u64 gpu_dump_va) -{ - WARN_ON(!kctx); - if (gpu_dump_va) - kbase_mem_free(kctx, gpu_dump_va); -} - -/** - * kbasep_hwcnt_backend_jm_destroy() - Destroy a JM backend. - * @backend: Pointer to JM backend to destroy. - * - * Can be safely called on a backend in any state of partial construction. - */ -static void kbasep_hwcnt_backend_jm_destroy(struct kbase_hwcnt_backend_jm *backend) -{ - if (!backend) - return; - - if (backend->kctx) { - struct kbase_context *kctx = backend->kctx; - struct kbase_device *kbdev = kctx->kbdev; - - if (backend->cpu_dump_va) - kbase_phy_alloc_mapping_put(kctx, backend->vmap); - - if (backend->gpu_dump_va) - kbasep_hwcnt_backend_jm_dump_free(kctx, backend->gpu_dump_va); - - kbasep_js_release_privileged_ctx(kbdev, kctx); - kbase_destroy_context(kctx); - } - - kfree(backend->to_user_buf); - - kfree(backend->to_user_block_states); - - kfree(backend); -} - -/** - * kbasep_hwcnt_backend_jm_create() - Create a JM backend. - * @info: Non-NULL pointer to backend info. - * @out_backend: Non-NULL pointer to where backend is stored on success. - * - * Return: 0 on success, else error code. - */ -static int kbasep_hwcnt_backend_jm_create(const struct kbase_hwcnt_backend_jm_info *info, - struct kbase_hwcnt_backend_jm **out_backend) -{ - int errcode; - struct kbase_device *kbdev; - struct kbase_hwcnt_backend_jm *backend = NULL; - size_t block_state_bytes; - - WARN_ON(!info); - WARN_ON(!out_backend); - - kbdev = info->kbdev; - - backend = kzalloc(sizeof(*backend), GFP_KERNEL); - if (!backend) - goto alloc_error; - - backend->info = info; - kbasep_hwcnt_backend_jm_init_layout(&info->hwcnt_gpu_info, &backend->phys_layout); - - backend->kctx = kbase_create_context(kbdev, true, - BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED, 0, NULL); - if (!backend->kctx) - goto alloc_error; - - kbasep_js_schedule_privileged_ctx(kbdev, backend->kctx); - - errcode = kbasep_hwcnt_backend_jm_dump_alloc(info, backend->kctx, &backend->gpu_dump_va); - if (errcode) - goto error; - - backend->cpu_dump_va = - kbase_phy_alloc_mapping_get(backend->kctx, backend->gpu_dump_va, &backend->vmap); - if (!backend->cpu_dump_va || !backend->vmap) - goto alloc_error; - - backend->to_user_buf = kzalloc(info->metadata->dump_buf_bytes, GFP_KERNEL); - if (!backend->to_user_buf) - goto alloc_error; - - block_state_bytes = backend->phys_layout.block_cnt * KBASE_HWCNT_BLOCK_STATE_BYTES * - KBASE_HWCNT_BLOCK_STATE_STRIDE; - - backend->to_user_block_states = kzalloc(block_state_bytes, GFP_KERNEL); - if (!backend->to_user_block_states) - goto alloc_error; - - memset(backend->to_user_block_states, 0, block_state_bytes); - - kbase_ccswe_init(&backend->ccswe_shader_cores); - backend->rate_listener.notify = kbasep_hwcnt_backend_jm_on_freq_change; - - *out_backend = backend; - return 0; - -alloc_error: - errcode = -ENOMEM; -error: - kbasep_hwcnt_backend_jm_destroy(backend); - return errcode; -} - -/* JM backend implementation of kbase_hwcnt_backend_metadata_fn */ -static const struct kbase_hwcnt_metadata * -kbasep_hwcnt_backend_jm_metadata(const struct kbase_hwcnt_backend_info *info) -{ - if (!info) - return NULL; - - return ((const struct kbase_hwcnt_backend_jm_info *)info)->metadata; -} - -/* JM backend implementation of kbase_hwcnt_backend_init_fn */ -static int kbasep_hwcnt_backend_jm_init(const struct kbase_hwcnt_backend_info *info, - struct kbase_hwcnt_backend **out_backend) -{ - int errcode; - struct kbase_hwcnt_backend_jm *backend = NULL; - - if (!info || !out_backend) - return -EINVAL; - - errcode = kbasep_hwcnt_backend_jm_create((const struct kbase_hwcnt_backend_jm_info *)info, - &backend); - if (errcode) - return errcode; - - *out_backend = (struct kbase_hwcnt_backend *)backend; - - return 0; -} - -/* JM backend implementation of kbase_hwcnt_backend_term_fn */ -static void kbasep_hwcnt_backend_jm_term(struct kbase_hwcnt_backend *backend) -{ - if (!backend) - return; - - kbasep_hwcnt_backend_jm_dump_disable(backend, NULL, NULL); - kbasep_hwcnt_backend_jm_destroy((struct kbase_hwcnt_backend_jm *)backend); -} - -static void kbasep_hwcnt_backend_jm_acquire(const struct kbase_hwcnt_backend *backend) -{ -} - -static void kbasep_hwcnt_backend_jm_release(const struct kbase_hwcnt_backend *backend) -{ -} - -/** - * kbasep_hwcnt_backend_jm_info_destroy() - Destroy a JM backend info. - * @info: Pointer to info to destroy. - * - * Can be safely called on a backend info in any state of partial construction. - */ -static void kbasep_hwcnt_backend_jm_info_destroy(const struct kbase_hwcnt_backend_jm_info *info) -{ - if (!info) - return; - - kbase_hwcnt_metadata_destroy(info->metadata); - kfree(info); -} - -/** - * kbasep_hwcnt_backend_jm_info_create() - Create a JM backend info. - * @kbdev: Non_NULL pointer to kbase device. - * @out_info: Non-NULL pointer to where info is stored on success. - * - * Return: 0 on success, else error code. - */ -static int kbasep_hwcnt_backend_jm_info_create(struct kbase_device *kbdev, - const struct kbase_hwcnt_backend_jm_info **out_info) -{ - int errcode = -ENOMEM; - struct kbase_hwcnt_backend_jm_info *info = NULL; - - WARN_ON(!kbdev); - WARN_ON(!out_info); - - info = kzalloc(sizeof(*info), GFP_KERNEL); - if (!info) - return errcode; - - info->kbdev = kbdev; - -#ifdef CONFIG_MALI_VALHALL_PRFCNT_SET_SECONDARY - info->counter_set = KBASE_HWCNT_SET_SECONDARY; -#elif defined(CONFIG_MALI_VALHALL_PRFCNT_SET_TERTIARY) - info->counter_set = KBASE_HWCNT_SET_TERTIARY; -#else - /* Default to primary */ - info->counter_set = KBASE_HWCNT_SET_PRIMARY; -#endif - - errcode = kbasep_hwcnt_backend_jm_gpu_info_init(kbdev, &info->hwcnt_gpu_info); - if (errcode) - goto error; - - errcode = kbase_hwcnt_jm_metadata_create(&info->hwcnt_gpu_info, info->counter_set, - &info->metadata, &info->dump_bytes); - if (errcode) - goto error; - - *out_info = info; - - return 0; -error: - kbasep_hwcnt_backend_jm_info_destroy(info); - return errcode; -} - -int kbase_hwcnt_backend_jm_create(struct kbase_device *kbdev, - struct kbase_hwcnt_backend_interface *iface) -{ - int errcode; - const struct kbase_hwcnt_backend_jm_info *info = NULL; - - if (!kbdev || !iface) - return -EINVAL; - - errcode = kbasep_hwcnt_backend_jm_info_create(kbdev, &info); - - if (errcode) - return errcode; - - iface->info = (struct kbase_hwcnt_backend_info *)info; - iface->metadata = kbasep_hwcnt_backend_jm_metadata; - iface->init = kbasep_hwcnt_backend_jm_init; - iface->term = kbasep_hwcnt_backend_jm_term; - iface->acquire = kbasep_hwcnt_backend_jm_acquire; - iface->release = kbasep_hwcnt_backend_jm_release; - iface->timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns; - iface->dump_enable = kbasep_hwcnt_backend_jm_dump_enable; - iface->dump_enable_nolock = kbasep_hwcnt_backend_jm_dump_enable_nolock; - iface->dump_disable = kbasep_hwcnt_backend_jm_dump_disable; - iface->dump_clear = kbasep_hwcnt_backend_jm_dump_clear; - iface->dump_request = kbasep_hwcnt_backend_jm_dump_request; - iface->dump_wait = kbasep_hwcnt_backend_jm_dump_wait; - iface->dump_get = kbasep_hwcnt_backend_jm_dump_get; - - return 0; -} - -void kbase_hwcnt_backend_jm_destroy(struct kbase_hwcnt_backend_interface *iface) -{ - if (!iface) - return; - - kbasep_hwcnt_backend_jm_info_destroy( - (const struct kbase_hwcnt_backend_jm_info *)iface->info); - memset(iface, 0, sizeof(*iface)); -} diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.h b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.h deleted file mode 100644 index 4a6293c25473..000000000000 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm.h +++ /dev/null @@ -1,58 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2018, 2020-2022 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Concrete implementation of mali_kbase_hwcnt_backend interface for JM - * backend. - */ - -#ifndef _KBASE_HWCNT_BACKEND_JM_H_ -#define _KBASE_HWCNT_BACKEND_JM_H_ - -#include "hwcnt/backend/mali_kbase_hwcnt_backend.h" - -struct kbase_device; - -/** - * kbase_hwcnt_backend_jm_create() - Create a JM hardware counter backend - * interface. - * @kbdev: Non-NULL pointer to kbase device. - * @iface: Non-NULL pointer to backend interface structure that is filled in - * on creation success. - * - * Calls to iface->dump_enable_nolock() require kbdev->hwaccess_lock held. - * - * Return: 0 on success, else error code. - */ -int kbase_hwcnt_backend_jm_create(struct kbase_device *kbdev, - struct kbase_hwcnt_backend_interface *iface); - -/** - * kbase_hwcnt_backend_jm_destroy() - Destroy a JM hardware counter backend - * interface. - * @iface: Pointer to interface to destroy. - * - * Can be safely called on an all-zeroed interface, or on an already destroyed - * interface. - */ -void kbase_hwcnt_backend_jm_destroy(struct kbase_hwcnt_backend_interface *iface); - -#endif /* _KBASE_HWCNT_BACKEND_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c deleted file mode 100644 index a68be8c64fd9..000000000000 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c +++ /dev/null @@ -1,841 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include - -#include -#include - -#include -#include -#include - -/* - * IDLE_BUFFER_EMPTY -> USER_DUMPING_BUFFER_EMPTY on dump_request. - * IDLE_BUFFER_EMPTY -> TIMER_DUMPING after - * hwcnt_backend_watchdog_timer_interval_ms - * milliseconds, if no dump_request has been - * called in the meantime. - * IDLE_BUFFER_FULL -> USER_DUMPING_BUFFER_FULL on dump_request. - * IDLE_BUFFER_FULL -> TIMER_DUMPING after - * hwcnt_backend_watchdog_timer_interval_ms - * milliseconds, if no dump_request has been - * called in the meantime. - * IDLE_BUFFER_FULL -> IDLE_BUFFER_EMPTY on dump_disable, upon discarding undumped - * counter values since the last dump_get. - * IDLE_BUFFER_EMPTY -> BUFFER_CLEARING on dump_clear, before calling job manager - * backend dump_clear. - * IDLE_BUFFER_FULL -> BUFFER_CLEARING on dump_clear, before calling job manager - * backend dump_clear. - * USER_DUMPING_BUFFER_EMPTY -> BUFFER_CLEARING on dump_clear, before calling job manager - * backend dump_clear. - * USER_DUMPING_BUFFER_FULL -> BUFFER_CLEARING on dump_clear, before calling job manager - * backend dump_clear. - * BUFFER_CLEARING -> IDLE_BUFFER_EMPTY on dump_clear, upon job manager backend - * dump_clear completion. - * TIMER_DUMPING -> IDLE_BUFFER_FULL on timer's callback completion. - * TIMER_DUMPING -> TIMER_DUMPING_USER_CLEAR on dump_clear, notifies the callback thread - * that there is no need for dumping the buffer - * anymore, and that the client will proceed - * clearing the buffer. - * TIMER_DUMPING_USER_CLEAR -> IDLE_BUFFER_EMPTY on timer's callback completion, when a user - * requested a dump_clear. - * TIMER_DUMPING -> TIMER_DUMPING_USER_REQUESTED on dump_request, when a client performs a - * dump request while the timer is dumping (the - * timer will perform the dump and (once - * completed) the client will retrieve the value - * from the buffer). - * TIMER_DUMPING_USER_REQUESTED -> IDLE_BUFFER_EMPTY on dump_get, when a timer completed and the - * user reads the periodic dump buffer. - * Any -> ERROR if the job manager backend returns an error - * (of any kind). - * USER_DUMPING_BUFFER_EMPTY -> IDLE_BUFFER_EMPTY on dump_get (performs get, ignores the - * periodic dump buffer and returns). - * USER_DUMPING_BUFFER_FULL -> IDLE_BUFFER_EMPTY on dump_get (performs get, accumulates with - * periodic dump buffer and returns). - */ - -/** enum backend_watchdog_state State used to synchronize timer callbacks with the main thread. - * @HWCNT_JM_WD_ERROR: Received an error from the job manager backend calls. - * @HWCNT_JM_WD_IDLE_BUFFER_EMPTY: Initial state. Watchdog timer enabled, periodic dump buffer is - * empty. - * @HWCNT_JM_WD_IDLE_BUFFER_FULL: Watchdog timer enabled, periodic dump buffer is full. - * @HWCNT_JM_WD_BUFFER_CLEARING: The client is performing a dump clear. A concurrent timer callback - * thread should just ignore and reschedule another callback in - * hwcnt_backend_watchdog_timer_interval_ms milliseconds. - * @HWCNT_JM_WD_TIMER_DUMPING: The timer ran out. The callback is performing a periodic dump. - * @HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED: While the timer is performing a periodic dump, user - * requested a dump. - * @HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR: While the timer is performing a dump, user requested a - * dump_clear. The timer has to complete the periodic dump - * and clear buffer (internal and job manager backend). - * @HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY: From IDLE state, user requested a dump. The periodic - * dump buffer is empty. - * @HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL: From IDLE state, user requested a dump. The periodic dump - * buffer is full. - * - * While the state machine is in HWCNT_JM_WD_TIMER_DUMPING*, only the timer callback thread is - * allowed to call the job manager backend layer. - */ -enum backend_watchdog_state { - HWCNT_JM_WD_ERROR, - HWCNT_JM_WD_IDLE_BUFFER_EMPTY, - HWCNT_JM_WD_IDLE_BUFFER_FULL, - HWCNT_JM_WD_BUFFER_CLEARING, - HWCNT_JM_WD_TIMER_DUMPING, - HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED, - HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR, - HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY, - HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL, -}; - -/** enum wd_init_state - State machine for initialization / termination of the backend resources - */ -enum wd_init_state { - HWCNT_JM_WD_INIT_START, - HWCNT_JM_WD_INIT_BACKEND = HWCNT_JM_WD_INIT_START, - HWCNT_JM_WD_INIT_ENABLE_MAP, - HWCNT_JM_WD_INIT_DUMP_BUFFER, - HWCNT_JM_WD_INIT_END -}; - -/** - * struct kbase_hwcnt_backend_jm_watchdog_info - Immutable information used to initialize an - * instance of the job manager watchdog backend. - * @jm_backend_iface: Hardware counter backend interface. This module extends - * this interface with a watchdog that performs regular - * dumps. The new interface this module provides complies - * with the old backend interface. - * @dump_watchdog_iface: Dump watchdog interface, used to periodically dump the - * hardware counter in case no reads are requested within - * a certain time, used to avoid hardware counter's buffer - * saturation. - * @watchdog_timer_interval_ms: Interval in milliseconds between hwcnt samples. - */ -struct kbase_hwcnt_backend_jm_watchdog_info { - struct kbase_hwcnt_backend_interface *jm_backend_iface; - struct kbase_hwcnt_watchdog_interface *dump_watchdog_iface; - u32 watchdog_timer_interval_ms; -}; - -/** - * struct kbase_hwcnt_backend_jm_watchdog - An instance of the job manager watchdog backend. - * @info: Immutable information used to create the job manager watchdog backend. - * @jm_backend: Job manager's backend internal state. To be passed as argument during parent calls. - * @timeout_ms: Time period in milliseconds for hardware counters dumping. - * @wd_dump_buffer: Used to store periodic dumps done by a timer callback function. Contents are - * valid in state %HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED, - * %HWCNT_JM_WD_IDLE_BUFFER_FULL or %HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL. - * @wd_enable_map: Watchdog backend internal buffer mask, initialized during dump_enable copying - * the enable_map passed as argument. - * @wd_dump_timestamp: Holds the dumping timestamp for potential future client dump_request, filled - * during watchdog timer dumps. - * @watchdog_complete: Used for synchronization between watchdog dumper thread and client calls. - * @locked: Members protected from concurrent access by different threads. - * @locked.watchdog_lock: Lock used to access fields within this struct (that require mutual - * exclusion). - * @locked.is_enabled: If true then the wrapped job manager hardware counter backend and the - * watchdog timer are both enabled. If false then both are disabled (or soon - * will be). Races between enable and disable have undefined behavior. - * @locked.state: State used to synchronize timer callbacks with the main thread. - */ -struct kbase_hwcnt_backend_jm_watchdog { - const struct kbase_hwcnt_backend_jm_watchdog_info *info; - struct kbase_hwcnt_backend *jm_backend; - u32 timeout_ms; - struct kbase_hwcnt_dump_buffer wd_dump_buffer; - struct kbase_hwcnt_enable_map wd_enable_map; - u64 wd_dump_timestamp; - struct completion watchdog_complete; - struct { - spinlock_t watchdog_lock; - bool is_enabled; - enum backend_watchdog_state state; - } locked; -}; - -/* timer's callback function */ -static void kbasep_hwcnt_backend_jm_watchdog_timer_callback(void *backend) -{ - struct kbase_hwcnt_backend_jm_watchdog *wd_backend = backend; - unsigned long flags; - bool wd_accumulate; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - if (!wd_backend->locked.is_enabled || wd_backend->locked.state == HWCNT_JM_WD_ERROR) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return; - } - - if (!(wd_backend->locked.state == HWCNT_JM_WD_IDLE_BUFFER_EMPTY || - wd_backend->locked.state == HWCNT_JM_WD_IDLE_BUFFER_FULL)) { - /*resetting the timer. Calling modify on a disabled timer enables it.*/ - wd_backend->info->dump_watchdog_iface->modify( - wd_backend->info->dump_watchdog_iface->timer, wd_backend->timeout_ms); - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return; - } - /*start performing the dump*/ - - /* if there has been a previous timeout use accumulating dump_get() - * otherwise use non-accumulating to overwrite buffer - */ - wd_accumulate = (wd_backend->locked.state == HWCNT_JM_WD_IDLE_BUFFER_FULL); - - wd_backend->locked.state = HWCNT_JM_WD_TIMER_DUMPING; - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - if (wd_backend->info->jm_backend_iface->dump_request(wd_backend->jm_backend) || - wd_backend->info->jm_backend_iface->dump_wait(wd_backend->jm_backend) || - wd_backend->info->jm_backend_iface->dump_get( - wd_backend->jm_backend, &wd_backend->wd_dump_buffer, &wd_backend->wd_enable_map, - wd_accumulate, &wd_backend->wd_dump_timestamp)) { - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - WARN_ON(wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING && - wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR && - wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED); - wd_backend->locked.state = HWCNT_JM_WD_ERROR; - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - /* Unblock user if it's waiting. */ - complete_all(&wd_backend->watchdog_complete); - return; - } - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - WARN_ON(wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING && - wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR && - wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED); - - if (wd_backend->locked.state == HWCNT_JM_WD_TIMER_DUMPING) { - /* If there is no user request/clear, transit to HWCNT_JM_WD_IDLE_BUFFER_FULL - * to indicate timer dump is done and the buffer is full. If state changed to - * HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED or - * HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR then user will transit the state - * machine to next state. - */ - wd_backend->locked.state = HWCNT_JM_WD_IDLE_BUFFER_FULL; - } - if (wd_backend->locked.state != HWCNT_JM_WD_ERROR && wd_backend->locked.is_enabled) { - /* reset the timer to schedule another callback. Calling modify on a - * disabled timer enables it. - */ - /*The spin lock needs to be held in case the client calls dump_enable*/ - wd_backend->info->dump_watchdog_iface->modify( - wd_backend->info->dump_watchdog_iface->timer, wd_backend->timeout_ms); - } - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - /* Unblock user if it's waiting. */ - complete_all(&wd_backend->watchdog_complete); -} - -/* helper methods, info structure creation and destruction*/ - -static struct kbase_hwcnt_backend_jm_watchdog_info * -kbasep_hwcnt_backend_jm_watchdog_info_create(struct kbase_hwcnt_backend_interface *backend_iface, - struct kbase_hwcnt_watchdog_interface *watchdog_iface, - u32 watchdog_timer_interval_ms) -{ - struct kbase_hwcnt_backend_jm_watchdog_info *const info = - kmalloc(sizeof(*info), GFP_KERNEL); - - if (!info) - return NULL; - - *info = (struct kbase_hwcnt_backend_jm_watchdog_info){ - .jm_backend_iface = backend_iface, - .dump_watchdog_iface = watchdog_iface, - .watchdog_timer_interval_ms = watchdog_timer_interval_ms - }; - - return info; -} - -/****** kbase_hwcnt_backend_interface implementation *******/ - -/* Job manager watchdog backend, implementation of kbase_hwcnt_backend_metadata_fn */ -static const struct kbase_hwcnt_metadata * -kbasep_hwcnt_backend_jm_watchdog_metadata(const struct kbase_hwcnt_backend_info *info) -{ - const struct kbase_hwcnt_backend_jm_watchdog_info *wd_info = (void *)info; - - if (WARN_ON(!info)) - return NULL; - - return wd_info->jm_backend_iface->metadata(wd_info->jm_backend_iface->info); -} - -static void -kbasep_hwcnt_backend_jm_watchdog_term_partial(struct kbase_hwcnt_backend_jm_watchdog *wd_backend, - enum wd_init_state state) -{ - if (!wd_backend) - return; - - WARN_ON(state > HWCNT_JM_WD_INIT_END); - - while (state-- > HWCNT_JM_WD_INIT_START) { - switch (state) { - case HWCNT_JM_WD_INIT_BACKEND: - wd_backend->info->jm_backend_iface->term(wd_backend->jm_backend); - break; - case HWCNT_JM_WD_INIT_ENABLE_MAP: - kbase_hwcnt_enable_map_free(&wd_backend->wd_enable_map); - break; - case HWCNT_JM_WD_INIT_DUMP_BUFFER: - kbase_hwcnt_dump_buffer_free(&wd_backend->wd_dump_buffer); - break; - case HWCNT_JM_WD_INIT_END: - break; - } - } - - kfree(wd_backend); -} - -static void kbasep_hwcnt_backend_jm_watchdog_acquire(const struct kbase_hwcnt_backend *backend) -{ -} - -static void kbasep_hwcnt_backend_jm_watchdog_release(const struct kbase_hwcnt_backend *backend) -{ -} - -/* Job manager watchdog backend, implementation of kbase_hwcnt_backend_term_fn - * Calling term does *not* destroy the interface - */ -static void kbasep_hwcnt_backend_jm_watchdog_term(struct kbase_hwcnt_backend *backend) -{ - struct kbase_hwcnt_backend_jm_watchdog *wd_backend = - (struct kbase_hwcnt_backend_jm_watchdog *)backend; - - if (!backend) - return; - - /* disable timer thread to avoid concurrent access to shared resources */ - wd_backend->info->dump_watchdog_iface->disable( - wd_backend->info->dump_watchdog_iface->timer); - - kbasep_hwcnt_backend_jm_watchdog_term_partial(wd_backend, HWCNT_JM_WD_INIT_END); -} - -/* Job manager watchdog backend, implementation of kbase_hwcnt_backend_init_fn */ -static int kbasep_hwcnt_backend_jm_watchdog_init(const struct kbase_hwcnt_backend_info *info, - struct kbase_hwcnt_backend **out_backend) -{ - int errcode = 0; - struct kbase_hwcnt_backend_jm_watchdog *wd_backend = NULL; - struct kbase_hwcnt_backend_jm_watchdog_info *const wd_info = (void *)info; - const struct kbase_hwcnt_backend_info *jm_info; - const struct kbase_hwcnt_metadata *metadata; - enum wd_init_state state = HWCNT_JM_WD_INIT_START; - - if (WARN_ON(!info) || WARN_ON(!out_backend)) - return -EINVAL; - - jm_info = wd_info->jm_backend_iface->info; - metadata = wd_info->jm_backend_iface->metadata(wd_info->jm_backend_iface->info); - - wd_backend = kmalloc(sizeof(*wd_backend), GFP_KERNEL); - if (!wd_backend) { - *out_backend = NULL; - return -ENOMEM; - } - - *wd_backend = (struct kbase_hwcnt_backend_jm_watchdog){ - .info = wd_info, - .timeout_ms = wd_info->watchdog_timer_interval_ms, - .locked = { .state = HWCNT_JM_WD_IDLE_BUFFER_EMPTY, .is_enabled = false } - }; - - while (state < HWCNT_JM_WD_INIT_END && !errcode) { - switch (state) { - case HWCNT_JM_WD_INIT_BACKEND: - errcode = wd_info->jm_backend_iface->init(jm_info, &wd_backend->jm_backend); - break; - case HWCNT_JM_WD_INIT_ENABLE_MAP: - errcode = - kbase_hwcnt_enable_map_alloc(metadata, &wd_backend->wd_enable_map); - break; - case HWCNT_JM_WD_INIT_DUMP_BUFFER: - errcode = kbase_hwcnt_dump_buffer_alloc(metadata, - &wd_backend->wd_dump_buffer); - break; - case HWCNT_JM_WD_INIT_END: - break; - } - if (!errcode) - state++; - } - - if (errcode) { - kbasep_hwcnt_backend_jm_watchdog_term_partial(wd_backend, state); - *out_backend = NULL; - return errcode; - } - - WARN_ON(state != HWCNT_JM_WD_INIT_END); - - spin_lock_init(&wd_backend->locked.watchdog_lock); - init_completion(&wd_backend->watchdog_complete); - - *out_backend = (struct kbase_hwcnt_backend *)wd_backend; - return 0; -} - -/* Job manager watchdog backend, implementation of timestamp_ns */ -static u64 kbasep_hwcnt_backend_jm_watchdog_timestamp_ns(struct kbase_hwcnt_backend *backend) -{ - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - - return wd_backend->info->jm_backend_iface->timestamp_ns(wd_backend->jm_backend); -} - -static int kbasep_hwcnt_backend_jm_watchdog_dump_enable_common( - struct kbase_hwcnt_backend_jm_watchdog *wd_backend, - const struct kbase_hwcnt_enable_map *enable_map, kbase_hwcnt_backend_dump_enable_fn enabler) -{ - int errcode = -EPERM; - unsigned long flags; - - if (WARN_ON(!wd_backend) || WARN_ON(!enable_map)) - return -EINVAL; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - /* If the backend is already enabled return an error */ - if (wd_backend->locked.is_enabled) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return -EPERM; - } - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - /*We copy the enable map into our watchdog backend copy, for future usage*/ - kbase_hwcnt_enable_map_copy(&wd_backend->wd_enable_map, enable_map); - - errcode = enabler(wd_backend->jm_backend, enable_map); - if (!errcode) { - /*Enable dump watchdog*/ - errcode = wd_backend->info->dump_watchdog_iface->enable( - wd_backend->info->dump_watchdog_iface->timer, wd_backend->timeout_ms, - kbasep_hwcnt_backend_jm_watchdog_timer_callback, wd_backend); - if (!errcode) { - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - WARN_ON(wd_backend->locked.is_enabled); - wd_backend->locked.is_enabled = true; - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - } else - /*Reverting the job manager backend back to disabled*/ - wd_backend->info->jm_backend_iface->dump_disable(wd_backend->jm_backend, - NULL, NULL); - } - - return errcode; -} - -/* Job manager watchdog backend, implementation of dump_enable */ -static int -kbasep_hwcnt_backend_jm_watchdog_dump_enable(struct kbase_hwcnt_backend *backend, - const struct kbase_hwcnt_enable_map *enable_map) -{ - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - - return kbasep_hwcnt_backend_jm_watchdog_dump_enable_common( - wd_backend, enable_map, wd_backend->info->jm_backend_iface->dump_enable); -} - -/* Job manager watchdog backend, implementation of dump_enable_nolock */ -static int -kbasep_hwcnt_backend_jm_watchdog_dump_enable_nolock(struct kbase_hwcnt_backend *backend, - const struct kbase_hwcnt_enable_map *enable_map) -{ - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - - return kbasep_hwcnt_backend_jm_watchdog_dump_enable_common( - wd_backend, enable_map, wd_backend->info->jm_backend_iface->dump_enable_nolock); -} - -/* Job manager watchdog backend, implementation of dump_disable */ -static void -kbasep_hwcnt_backend_jm_watchdog_dump_disable(struct kbase_hwcnt_backend *backend, - struct kbase_hwcnt_dump_buffer *dump_buffer, - const struct kbase_hwcnt_enable_map *buf_enable_map) -{ - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - unsigned long flags; - - if (WARN_ON(!backend)) - return; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - if (!wd_backend->locked.is_enabled) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return; - } - - wd_backend->locked.is_enabled = false; - - /* Discard undumped counter values since the last dump_get. */ - if (wd_backend->locked.state == HWCNT_JM_WD_IDLE_BUFFER_FULL) - wd_backend->locked.state = HWCNT_JM_WD_IDLE_BUFFER_EMPTY; - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - wd_backend->info->dump_watchdog_iface->disable( - wd_backend->info->dump_watchdog_iface->timer); - - wd_backend->info->jm_backend_iface->dump_disable(wd_backend->jm_backend, dump_buffer, - buf_enable_map); -} - -/* Job manager watchdog backend, implementation of dump_clear */ -static int kbasep_hwcnt_backend_jm_watchdog_dump_clear(struct kbase_hwcnt_backend *backend) -{ - int errcode = -EPERM; - bool clear_wd_wait_completion = false; - unsigned long flags; - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - - if (WARN_ON(!backend)) - return -EINVAL; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - if (!wd_backend->locked.is_enabled) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return -EPERM; - } - - switch (wd_backend->locked.state) { - case HWCNT_JM_WD_IDLE_BUFFER_FULL: - case HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL: - case HWCNT_JM_WD_IDLE_BUFFER_EMPTY: - case HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY: - wd_backend->locked.state = HWCNT_JM_WD_BUFFER_CLEARING; - errcode = 0; - break; - case HWCNT_JM_WD_TIMER_DUMPING: - /* The timer asked for a dump request, when complete, the job manager backend - * buffer will be zero - */ - clear_wd_wait_completion = true; - /* This thread will have to wait for the callback to terminate and then call a - * dump_clear on the job manager backend. We change the state to - * HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR to notify the callback thread there is - * no more need to dump the buffer (since we will clear it right after anyway). - * We set up a wait queue to synchronize with the callback. - */ - reinit_completion(&wd_backend->watchdog_complete); - wd_backend->locked.state = HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR; - errcode = 0; - break; - default: - errcode = -EPERM; - break; - } - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - if (!errcode) { - if (clear_wd_wait_completion) { - /* Waiting for the callback to finish */ - wait_for_completion(&wd_backend->watchdog_complete); - } - - /* Clearing job manager backend buffer */ - errcode = wd_backend->info->jm_backend_iface->dump_clear(wd_backend->jm_backend); - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - WARN_ON(wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_CLEAR && - wd_backend->locked.state != HWCNT_JM_WD_BUFFER_CLEARING && - wd_backend->locked.state != HWCNT_JM_WD_ERROR); - - WARN_ON(!wd_backend->locked.is_enabled); - - if (!errcode && wd_backend->locked.state != HWCNT_JM_WD_ERROR) { - /* Setting the internal buffer state to EMPTY */ - wd_backend->locked.state = HWCNT_JM_WD_IDLE_BUFFER_EMPTY; - /* Resetting the timer. Calling modify on a disabled timer - * enables it. - */ - wd_backend->info->dump_watchdog_iface->modify( - wd_backend->info->dump_watchdog_iface->timer, - wd_backend->timeout_ms); - } else { - wd_backend->locked.state = HWCNT_JM_WD_ERROR; - errcode = -EPERM; - } - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - } - - return errcode; -} - -/* Job manager watchdog backend, implementation of dump_request */ -static int kbasep_hwcnt_backend_jm_watchdog_dump_request(struct kbase_hwcnt_backend *backend) -{ - bool call_dump_request = false; - int errcode = 0; - unsigned long flags; - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - - if (WARN_ON(!backend)) - return -EINVAL; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - if (!wd_backend->locked.is_enabled) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return -EPERM; - } - - switch (wd_backend->locked.state) { - case HWCNT_JM_WD_IDLE_BUFFER_EMPTY: - /* progressing the state to avoid callbacks running while calling the job manager - * backend - */ - wd_backend->locked.state = HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY; - call_dump_request = true; - break; - case HWCNT_JM_WD_IDLE_BUFFER_FULL: - wd_backend->locked.state = HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL; - call_dump_request = true; - break; - case HWCNT_JM_WD_TIMER_DUMPING: - /* On the next client call (dump_wait) the thread will have to wait for the - * callback to finish the dumping. - * We set up a wait queue to synchronize with the callback. - */ - reinit_completion(&wd_backend->watchdog_complete); - wd_backend->locked.state = HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED; - break; - default: - errcode = -EPERM; - break; - } - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - if (call_dump_request) { - errcode = wd_backend->info->jm_backend_iface->dump_request(wd_backend->jm_backend); - if (!errcode) { - /*resetting the timer. Calling modify on a disabled timer enables it*/ - wd_backend->info->dump_watchdog_iface->modify( - wd_backend->info->dump_watchdog_iface->timer, - wd_backend->timeout_ms); - } else { - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - WARN_ON(!wd_backend->locked.is_enabled); - wd_backend->locked.state = HWCNT_JM_WD_ERROR; - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - } - } - - return errcode; -} - -/* Job manager watchdog backend, implementation of dump_wait */ -static int kbasep_hwcnt_backend_jm_watchdog_dump_wait(struct kbase_hwcnt_backend *backend) -{ - int errcode = -EPERM; - bool wait_for_auto_dump = false, wait_for_user_dump = false; - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - unsigned long flags; - - if (WARN_ON(!backend)) - return -EINVAL; - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - if (!wd_backend->locked.is_enabled) { - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - return -EPERM; - } - - switch (wd_backend->locked.state) { - case HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED: - wait_for_auto_dump = true; - errcode = 0; - break; - case HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY: - case HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL: - wait_for_user_dump = true; - errcode = 0; - break; - default: - errcode = -EPERM; - break; - } - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - if (wait_for_auto_dump) - wait_for_completion(&wd_backend->watchdog_complete); - else if (wait_for_user_dump) { - errcode = wd_backend->info->jm_backend_iface->dump_wait(wd_backend->jm_backend); - if (errcode) { - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - WARN_ON(!wd_backend->locked.is_enabled); - wd_backend->locked.state = HWCNT_JM_WD_ERROR; - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - } - } - - return errcode; -} - -/* Job manager watchdog backend, implementation of dump_get */ -static int kbasep_hwcnt_backend_jm_watchdog_dump_get( - struct kbase_hwcnt_backend *backend, struct kbase_hwcnt_dump_buffer *dump_buffer, - const struct kbase_hwcnt_enable_map *enable_map, bool accumulate, u64 *dump_time_ns) -{ - bool call_dump_get = false; - struct kbase_hwcnt_backend_jm_watchdog *const wd_backend = (void *)backend; - unsigned long flags; - int errcode = 0; - - if (WARN_ON(!backend) || WARN_ON(!dump_buffer) || WARN_ON(!enable_map) || - WARN_ON(!dump_time_ns)) - return -EINVAL; - - /* The resultant contents of the dump buffer are only well defined if a prior - * call to dump_wait returned successfully, and a new dump has not yet been - * requested by a call to dump_request. - */ - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - switch (wd_backend->locked.state) { - case HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED: - /*we assume dump_wait has been called and completed successfully*/ - if (accumulate) - kbase_hwcnt_dump_buffer_accumulate(dump_buffer, &wd_backend->wd_dump_buffer, - enable_map); - else - kbase_hwcnt_dump_buffer_copy(dump_buffer, &wd_backend->wd_dump_buffer, - enable_map); - - /*use state to indicate the the buffer is now empty*/ - wd_backend->locked.state = HWCNT_JM_WD_IDLE_BUFFER_EMPTY; - /* Retrieve timing information from previous dump_request */ - *dump_time_ns = wd_backend->wd_dump_timestamp; - break; - case HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL: - /*accumulate or copy watchdog data to user buffer first so that dump_get can set - * the header correctly - */ - if (accumulate) - kbase_hwcnt_dump_buffer_accumulate(dump_buffer, &wd_backend->wd_dump_buffer, - enable_map); - else - kbase_hwcnt_dump_buffer_copy(dump_buffer, &wd_backend->wd_dump_buffer, - enable_map); - - /*accumulate backend data into user buffer on top of watchdog data*/ - accumulate = true; - call_dump_get = true; - break; - case HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY: - call_dump_get = true; - break; - default: - errcode = -EPERM; - break; - } - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - - if (call_dump_get && !errcode) { - /*we just dump the job manager backend into the user buffer, following - *accumulate flag - */ - errcode = wd_backend->info->jm_backend_iface->dump_get( - wd_backend->jm_backend, dump_buffer, enable_map, accumulate, dump_time_ns); - - spin_lock_irqsave(&wd_backend->locked.watchdog_lock, flags); - - WARN_ON(wd_backend->locked.state != HWCNT_JM_WD_USER_DUMPING_BUFFER_EMPTY && - wd_backend->locked.state != HWCNT_JM_WD_USER_DUMPING_BUFFER_FULL && - wd_backend->locked.state != HWCNT_JM_WD_TIMER_DUMPING_USER_REQUESTED); - - if (!errcode) - wd_backend->locked.state = HWCNT_JM_WD_IDLE_BUFFER_EMPTY; - else - wd_backend->locked.state = HWCNT_JM_WD_ERROR; - - spin_unlock_irqrestore(&wd_backend->locked.watchdog_lock, flags); - } - - return errcode; -} - -/* exposed methods */ - -int kbase_hwcnt_backend_jm_watchdog_create(struct kbase_hwcnt_backend_interface *backend_iface, - struct kbase_hwcnt_watchdog_interface *watchdog_iface, - struct kbase_hwcnt_backend_interface *out_iface, - u32 watchdog_timer_interval_ms) -{ - struct kbase_hwcnt_backend_jm_watchdog_info *info = NULL; - - if (WARN_ON(!backend_iface) || WARN_ON(!watchdog_iface) || WARN_ON(!out_iface)) - return -EINVAL; - - info = kbasep_hwcnt_backend_jm_watchdog_info_create(backend_iface, watchdog_iface, - watchdog_timer_interval_ms); - if (!info) - return -ENOMEM; - - /*linking the info table with the output iface, to allow the callbacks below to access the - *info object later on - */ - *out_iface = (struct kbase_hwcnt_backend_interface){ - .info = (void *)info, - .metadata = kbasep_hwcnt_backend_jm_watchdog_metadata, - .init = kbasep_hwcnt_backend_jm_watchdog_init, - .term = kbasep_hwcnt_backend_jm_watchdog_term, - .acquire = kbasep_hwcnt_backend_jm_watchdog_acquire, - .release = kbasep_hwcnt_backend_jm_watchdog_release, - .timestamp_ns = kbasep_hwcnt_backend_jm_watchdog_timestamp_ns, - .dump_enable = kbasep_hwcnt_backend_jm_watchdog_dump_enable, - .dump_enable_nolock = kbasep_hwcnt_backend_jm_watchdog_dump_enable_nolock, - .dump_disable = kbasep_hwcnt_backend_jm_watchdog_dump_disable, - .dump_clear = kbasep_hwcnt_backend_jm_watchdog_dump_clear, - .dump_request = kbasep_hwcnt_backend_jm_watchdog_dump_request, - .dump_wait = kbasep_hwcnt_backend_jm_watchdog_dump_wait, - .dump_get = kbasep_hwcnt_backend_jm_watchdog_dump_get - }; - - /*registering watchdog backend module methods on the output interface*/ - - return 0; -} - -void kbase_hwcnt_backend_jm_watchdog_destroy(struct kbase_hwcnt_backend_interface *iface) -{ - if (!iface || !iface->info) - return; - - kfree((struct kbase_hwcnt_backend_jm_watchdog_info *)iface->info); - - /*blanking the watchdog backend interface*/ - memset(iface, 0, sizeof(*iface)); -} diff --git a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.h b/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.h deleted file mode 100644 index d5ce27685186..000000000000 --- a/drivers/gpu/arm/valhall/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.h +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Concrete implementation of mali_kbase_hwcnt_backend interface for job manager - * backend. This module functionally interleaves between the hardware counter - * (hwcnt_accumulator) module (the interface consumer) and the job manager - * backend module (hwcnt_backend_jm). This module provides buffering - * functionality for the dumping requests requested by the hwcnt_accumulator - * consumer. This module is NOT multi-thread safe. The programmer must - * ensure the exposed methods are called by at most one thread at any time. - */ - -#ifndef _KBASE_HWCNT_BACKEND_JM_WATCHDOG_H_ -#define _KBASE_HWCNT_BACKEND_JM_WATCHDOG_H_ - -#include -#include - -/** - * kbase_hwcnt_backend_jm_watchdog_create() - Create a job manager hardware counter watchdog - * backend interface. - * @backend_iface: Non-NULL pointer to the backend interface structure that this module will - * extend. - * @watchdog_iface: Non-NULL pointer to an hardware counter watchdog interface. - * @out_iface: Non-NULL pointer to backend interface structure that is filled in - * on creation success. - * @watchdog_timer_interval_ms: Interval in milliseconds between hwcnt samples. - * - * Calls to out_iface->dump_enable_nolock() require kbdev->hwaccess_lock held. - * - * Return: 0 on success, error otherwise. - */ -int kbase_hwcnt_backend_jm_watchdog_create(struct kbase_hwcnt_backend_interface *backend_iface, - struct kbase_hwcnt_watchdog_interface *watchdog_iface, - struct kbase_hwcnt_backend_interface *out_iface, - u32 watchdog_timer_interval_ms); - -/** - * kbase_hwcnt_backend_jm_watchdog_destroy() - Destroy a job manager hardware counter watchdog - * backend interface. - * @iface: Pointer to interface to destroy. - * - * Can be safely called on an all-zeroed interface, or on an already destroyed - * interface. - */ -void kbase_hwcnt_backend_jm_watchdog_destroy(struct kbase_hwcnt_backend_interface *iface); - -#endif /* _KBASE_HWCNT_BACKEND_JM_WATCHDOG_H_ */ diff --git a/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if.h b/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if.h index 501c0087b7e6..f8b12c12a4ef 100644 --- a/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if.h +++ b/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2021-2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -33,6 +33,15 @@ */ struct kbase_hwcnt_watchdog_info; +/* + * Enum describing different types of watchdog disable calls. + */ +enum kbase_hwcnt_watchdog_disable_type { + KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_BLOCK, + KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_NOT_BLOCK, + KBASE_HWCNT_WATCHDOG_DISABLE_TYPE_CNT +}; + /** * typedef kbase_hwcnt_watchdog_callback_fn - Callback function when watchdog timer is done * @@ -59,8 +68,10 @@ typedef int kbase_hwcnt_watchdog_enable_fn(const struct kbase_hwcnt_watchdog_inf * typedef kbase_hwcnt_watchdog_disable_fn - Disable watchdog timer * * @timer: Non-NULL pointer to a watchdog timer interface context + * @type: Disable type to use */ -typedef void kbase_hwcnt_watchdog_disable_fn(const struct kbase_hwcnt_watchdog_info *timer); +typedef void kbase_hwcnt_watchdog_disable_fn(const struct kbase_hwcnt_watchdog_info *timer, + enum kbase_hwcnt_watchdog_disable_type type); /** * typedef kbase_hwcnt_watchdog_modify_fn - Modify watchdog timer's timeout diff --git a/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if_timer.c b/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if_timer.c index 4caa832cd587..eae1cf6c0aae 100644 --- a/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if_timer.c +++ b/drivers/gpu/arm/valhall/hwcnt/mali_kbase_hwcnt_watchdog_if_timer.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2021-2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -57,7 +57,8 @@ static void kbasep_hwcnt_watchdog_callback(struct work_struct *const work) struct kbase_hwcnt_watchdog_if_timer_info *const info = container_of(work, struct kbase_hwcnt_watchdog_if_timer_info, dwork.work); - if (info->callback) + /* Issue the callback only if timer is enabled. */ + if (info->callback && info->timer_enabled) info->callback(info->user_data); } @@ -80,7 +81,8 @@ static int kbasep_hwcnt_watchdog_if_timer_enable( } static void -kbasep_hwcnt_watchdog_if_timer_disable(const struct kbase_hwcnt_watchdog_info *const timer) +kbasep_hwcnt_watchdog_if_timer_disable(const struct kbase_hwcnt_watchdog_info *const timer, + enum kbase_hwcnt_watchdog_disable_type type) { struct kbase_hwcnt_watchdog_if_timer_info *const timer_info = (void *)timer; @@ -90,7 +92,16 @@ kbasep_hwcnt_watchdog_if_timer_disable(const struct kbase_hwcnt_watchdog_info *c if (!timer_info->timer_enabled) return; - cancel_delayed_work_sync(&timer_info->dwork); + /* Cancel any pending work. + * Use a blocking call if the caller can afford to wait. + * In an opposite case (KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_NOT_BLOCK), + * timer callback will be effectively canceled by setting timer_enabled to false. + * This will prevent the timer callback from being issued and from rescheduling + * itself. + */ + if (type == KBASE_HWCNT_WATCHDOG_DISABLE_SHOULD_BLOCK) + cancel_delayed_work_sync(&timer_info->dwork); + timer_info->timer_enabled = false; } @@ -100,7 +111,7 @@ kbasep_hwcnt_watchdog_if_timer_modify(const struct kbase_hwcnt_watchdog_info *co { struct kbase_hwcnt_watchdog_if_timer_info *const timer_info = (void *)timer; - if (WARN_ON(!timer) || WARN_ON(!timer_info->timer_enabled)) + if (WARN_ON(!timer) || !timer_info->timer_enabled) return; mod_delayed_work(timer_info->workq, &timer_info->dwork, msecs_to_jiffies(delay_ms)); @@ -118,6 +129,7 @@ void kbase_hwcnt_watchdog_if_timer_destroy(struct kbase_hwcnt_watchdog_interface if (WARN_ON(!timer_info)) return; + cancel_delayed_work_sync(&timer_info->dwork); destroy_workqueue(timer_info->workq); kfree(timer_info); diff --git a/drivers/gpu/arm/valhall/ipa/Kbuild b/drivers/gpu/arm/valhall/ipa/Kbuild index 9d8a79d8dc29..968157446956 100644 --- a/drivers/gpu/arm/valhall/ipa/Kbuild +++ b/drivers/gpu/arm/valhall/ipa/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2016-2018, 2020-2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2016-2018, 2020-2021, 2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -20,16 +20,8 @@ valhall_kbase-y += \ ipa/mali_kbase_ipa_simple.o \ - ipa/mali_kbase_ipa.o + ipa/mali_kbase_ipa.o \ + ipa/backend/mali_kbase_ipa_counter_csf.o \ + ipa/backend/mali_kbase_ipa_counter_common_csf.o valhall_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o - -ifeq ($(MALI_USE_CSF),1) - valhall_kbase-y += \ - ipa/backend/mali_kbase_ipa_counter_csf.o \ - ipa/backend/mali_kbase_ipa_counter_common_csf.o -else - valhall_kbase-y += \ - ipa/backend/mali_kbase_ipa_counter_jm.o \ - ipa/backend/mali_kbase_ipa_counter_common_jm.o -endif diff --git a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.c b/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.c deleted file mode 100644 index a716e15b3df3..000000000000 --- a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.c +++ /dev/null @@ -1,339 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include "mali_kbase_ipa_counter_common_jm.h" -#include "ipa/mali_kbase_ipa_debugfs.h" - -#define DEFAULT_SCALING_FACTOR 5 - -/* If the value of GPU_ACTIVE is below this, use the simple model - * instead, to avoid extrapolating small amounts of counter data across - * large sample periods. - */ -#define DEFAULT_MIN_SAMPLE_CYCLES 10000 - -/** - * kbase_ipa_read_hwcnt() - read a counter value - * @model_data: pointer to model data - * @offset: offset, in bytes, into vinstr buffer - * - * Return: A 32-bit counter value. Range: 0 < value < 2^27 (worst case would be - * incrementing every cycle over a ~100ms sample period at a high frequency, - * e.g. 1 GHz: 2^30 * 0.1seconds ~= 2^27. - */ -static inline u32 kbase_ipa_read_hwcnt(struct kbase_ipa_model_vinstr_data *model_data, u32 offset) -{ - u8 *p = (u8 *)model_data->dump_buf.dump_buf; - u64 val = *(u64 *)&p[offset]; - - return (val > U32_MAX) ? U32_MAX : (u32)val; -} - -static inline s64 kbase_ipa_add_saturate(s64 a, s64 b) -{ - s64 rtn; - - if (a > 0 && (S64_MAX - a) < b) - rtn = S64_MAX; - else if (a < 0 && (S64_MIN - a) > b) - rtn = S64_MIN; - else - rtn = a + b; - - return rtn; -} - -s64 kbase_ipa_sum_all_shader_cores(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter) -{ - struct kbase_device *kbdev = model_data->kbdev; - u64 core_mask; - u32 base = 0; - s64 ret = 0; - - core_mask = kbdev->gpu_props.coherency_info.group.core_mask; - while (core_mask != 0ull) { - if ((core_mask & 1ull) != 0ull) { - /* 0 < counter_value < 2^27 */ - u32 counter_value = kbase_ipa_read_hwcnt(model_data, base + counter); - - /* 0 < ret < 2^27 * max_num_cores = 2^32 */ - ret = kbase_ipa_add_saturate(ret, counter_value); - } - base += KBASE_IPA_NR_BYTES_PER_BLOCK; - core_mask >>= 1; - } - - /* Range: -2^54 < ret * coeff < 2^54 */ - return ret * coeff; -} - -s64 kbase_ipa_sum_all_memsys_blocks(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter) -{ - struct kbase_device *kbdev = model_data->kbdev; - const u32 num_blocks = kbdev->gpu_props.num_l2_slices; - u32 base = 0; - s64 ret = 0; - u32 i; - - for (i = 0; i < num_blocks; i++) { - /* 0 < counter_value < 2^27 */ - u32 counter_value = kbase_ipa_read_hwcnt(model_data, base + counter); - - /* 0 < ret < 2^27 * max_num_memsys_blocks = 2^29 */ - ret = kbase_ipa_add_saturate(ret, counter_value); - base += KBASE_IPA_NR_BYTES_PER_BLOCK; - } - - /* Range: -2^51 < ret * coeff < 2^51 */ - return ret * coeff; -} - -s64 kbase_ipa_single_counter(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, u32 counter) -{ - /* Range: 0 < counter_value < 2^27 */ - const u32 counter_value = kbase_ipa_read_hwcnt(model_data, counter); - - /* Range: -2^49 < ret < 2^49 */ - return counter_value * (s64)coeff; -} - -int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) -{ - int errcode; - struct kbase_device *kbdev = model_data->kbdev; - struct kbase_hwcnt_virtualizer *hvirt = kbdev->hwcnt_gpu_virt; - struct kbase_hwcnt_enable_map enable_map; - const struct kbase_hwcnt_metadata *metadata = kbase_hwcnt_virtualizer_metadata(hvirt); - - if (!metadata) - return -1; - - errcode = kbase_hwcnt_enable_map_alloc(metadata, &enable_map); - if (errcode) { - dev_err(kbdev->dev, "Failed to allocate IPA enable map"); - return errcode; - } - - kbase_hwcnt_enable_map_enable_all(&enable_map); - - /* Disable cycle counter only. */ - enable_map.clk_enable_map = 0; - - errcode = kbase_hwcnt_virtualizer_client_create(hvirt, &enable_map, &model_data->hvirt_cli); - kbase_hwcnt_enable_map_free(&enable_map); - if (errcode) { - dev_err(kbdev->dev, "Failed to register IPA with virtualizer"); - model_data->hvirt_cli = NULL; - return errcode; - } - - errcode = kbase_hwcnt_dump_buffer_alloc(metadata, &model_data->dump_buf); - if (errcode) { - dev_err(kbdev->dev, "Failed to allocate IPA dump buffer"); - kbase_hwcnt_virtualizer_client_destroy(model_data->hvirt_cli); - model_data->hvirt_cli = NULL; - return errcode; - } - - return 0; -} - -void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) -{ - if (model_data->hvirt_cli) { - kbase_hwcnt_virtualizer_client_destroy(model_data->hvirt_cli); - kbase_hwcnt_dump_buffer_free(&model_data->dump_buf); - model_data->hvirt_cli = NULL; - } -} - -int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) -{ - struct kbase_ipa_model_vinstr_data *model_data = - (struct kbase_ipa_model_vinstr_data *)model->model_data; - s64 energy = 0; - size_t i; - u64 coeff = 0, coeff_mul = 0; - u64 start_ts_ns, end_ts_ns; - u32 active_cycles; - int err = 0; - - err = kbase_hwcnt_virtualizer_client_dump(model_data->hvirt_cli, &start_ts_ns, &end_ts_ns, - &model_data->dump_buf); - if (err) - goto err0; - - /* Range: 0 (GPU not used at all), to the max sampling interval, say - * 1s, * max GPU frequency (GPU 100% utilized). - * 0 <= active_cycles <= 1 * ~2GHz - * 0 <= active_cycles < 2^31 - */ - active_cycles = model_data->get_active_cycles(model_data); - - if (active_cycles < (u32)max(model_data->min_sample_cycles, 0)) { - err = -ENODATA; - goto err0; - } - - /* Range: 1 <= active_cycles < 2^31 */ - active_cycles = max(1u, active_cycles); - - /* Range of 'energy' is +/- 2^54 * number of IPA groups (~8), so around - * -2^57 < energy < 2^57 - */ - for (i = 0; i < model_data->groups_def_num; i++) { - const struct kbase_ipa_group *group = &model_data->groups_def[i]; - s32 coeff = model_data->group_values[i]; - s64 group_energy = group->op(model_data, coeff, group->counter_block_offset); - - energy = kbase_ipa_add_saturate(energy, group_energy); - } - - /* Range: 0 <= coeff < 2^57 */ - if (energy > 0) - coeff = (u64)energy; - - /* Range: 0 <= coeff < 2^57 (because active_cycles >= 1). However, this - * can be constrained further: Counter values can only be increased by - * a theoretical maximum of about 64k per clock cycle. Beyond this, - * we'd have to sample every 1ms to avoid them overflowing at the - * lowest clock frequency (say 100MHz). Therefore, we can write the - * range of 'coeff' in terms of active_cycles: - * - * coeff = SUM(coeffN * counterN * num_cores_for_counterN) - * coeff <= SUM(coeffN * counterN) * max_num_cores - * coeff <= num_IPA_groups * max_coeff * max_counter * max_num_cores - * (substitute max_counter = 2^16 * active_cycles) - * coeff <= num_IPA_groups * max_coeff * 2^16 * active_cycles * max_num_cores - * coeff <= 2^3 * 2^22 * 2^16 * active_cycles * 2^5 - * coeff <= 2^46 * active_cycles - * - * So after the division: 0 <= coeff <= 2^46 - */ - coeff = div_u64(coeff, active_cycles); - - /* Not all models were derived at the same reference voltage. Voltage - * scaling is done by multiplying by V^2, so we need to *divide* by - * Vref^2 here. - * Range: 0 <= coeff <= 2^49 - */ - coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); - /* Range: 0 <= coeff <= 2^52 */ - coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); - - /* Scale by user-specified integer factor. - * Range: 0 <= coeff_mul < 2^57 - */ - coeff_mul = coeff * (u64)model_data->scaling_factor; - - /* The power models have results with units - * mW/(MHz V^2), i.e. nW/(Hz V^2). With precision of 1/1000000, this - * becomes fW/(Hz V^2), which are the units of coeff_mul. However, - * kbase_scale_dynamic_power() expects units of pW/(Hz V^2), so divide - * by 1000. - * Range: 0 <= coeff_mul < 2^47 - */ - coeff_mul = div_u64(coeff_mul, 1000u); - -err0: - /* Clamp to a sensible range - 2^16 gives about 14W at 400MHz/750mV */ - *coeffp = clamp(coeff_mul, (u64)0, (u64)1 << 16); - return err; -} - -void kbase_ipa_vinstr_reset_data(struct kbase_ipa_model *model) -{ - CSTD_UNUSED(model); - /* Currently not implemented */ - WARN_ON_ONCE(1); -} - -int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, - const struct kbase_ipa_group *ipa_groups_def, - size_t ipa_group_size, - kbase_ipa_get_active_cycles_callback get_active_cycles, - s32 reference_voltage) -{ - int err = 0; - size_t i; - struct kbase_ipa_model_vinstr_data *model_data; - - if (!model || !ipa_groups_def || !ipa_group_size || !get_active_cycles) - return -EINVAL; - - model_data = kzalloc(sizeof(*model_data), GFP_KERNEL); - if (!model_data) - return -ENOMEM; - - model_data->kbdev = model->kbdev; - model_data->groups_def = ipa_groups_def; - model_data->groups_def_num = ipa_group_size; - model_data->get_active_cycles = get_active_cycles; - - model->model_data = (void *)model_data; - - for (i = 0; i < model_data->groups_def_num; ++i) { - const struct kbase_ipa_group *group = &model_data->groups_def[i]; - - model_data->group_values[i] = group->default_value; - err = kbase_ipa_model_add_param_s32(model, group->name, - &model_data->group_values[i], 1, false); - if (err) - goto exit; - } - - model_data->scaling_factor = DEFAULT_SCALING_FACTOR; - err = kbase_ipa_model_add_param_s32(model, "scale", &model_data->scaling_factor, 1, false); - if (err) - goto exit; - - model_data->min_sample_cycles = DEFAULT_MIN_SAMPLE_CYCLES; - err = kbase_ipa_model_add_param_s32(model, "min_sample_cycles", - &model_data->min_sample_cycles, 1, false); - if (err) - goto exit; - - model_data->reference_voltage = reference_voltage; - err = kbase_ipa_model_add_param_s32(model, "reference_voltage", - &model_data->reference_voltage, 1, false); - if (err) - goto exit; - - err = kbase_ipa_attach_vinstr(model_data); - -exit: - if (err) { - kbase_ipa_model_param_free_all(model); - kfree(model_data); - } - return err; -} - -void kbase_ipa_vinstr_common_model_term(struct kbase_ipa_model *model) -{ - struct kbase_ipa_model_vinstr_data *model_data = - (struct kbase_ipa_model_vinstr_data *)model->model_data; - - kbase_ipa_detach_vinstr(model_data); - kfree(model_data); -} diff --git a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.h b/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.h deleted file mode 100644 index d1d1f7d3b57f..000000000000 --- a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_common_jm.h +++ /dev/null @@ -1,227 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_IPA_COUNTER_COMMON_JM_H_ -#define _KBASE_IPA_COUNTER_COMMON_JM_H_ - -#include "mali_kbase.h" -#include "hwcnt/mali_kbase_hwcnt_virtualizer.h" -#include "hwcnt/mali_kbase_hwcnt_types.h" - -/* Maximum number of IPA groups for an IPA model. */ -#define KBASE_IPA_MAX_GROUP_DEF_NUM 16 - -/* Number of bytes per hardware counter in a vinstr_buffer. */ -#define KBASE_IPA_NR_BYTES_PER_CNT (sizeof(u64)) - -/* Number of hardware counters per block in a vinstr_buffer. */ -#define KBASE_IPA_NR_CNT_PER_BLOCK 64 - -/* Number of bytes per block in a vinstr_buffer. */ -#define KBASE_IPA_NR_BYTES_PER_BLOCK (KBASE_IPA_NR_CNT_PER_BLOCK * KBASE_IPA_NR_BYTES_PER_CNT) - -struct kbase_ipa_model_vinstr_data; - -typedef u32 kbase_ipa_get_active_cycles_callback(struct kbase_ipa_model_vinstr_data *); - -/** - * struct kbase_ipa_model_vinstr_data - IPA context per device - * @kbdev: pointer to kbase device - * @group_values: values of coefficients for IPA groups - * @groups_def: Array of IPA groups. - * @groups_def_num: Number of elements in the array of IPA groups. - * @get_active_cycles: Callback to return number of active cycles during - * counter sample period - * @hvirt_cli: hardware counter virtualizer client handle - * @dump_buf: buffer to dump hardware counters onto - * @reference_voltage: voltage, in mV, of the operating point used when - * deriving the power model coefficients. Range approx - * 0.1V - 5V (~= 8V): 2^7 <= reference_voltage <= 2^13 - * @scaling_factor: User-specified power scaling factor. This is an - * integer, which is multiplied by the power coefficient - * just before OPP scaling. - * Range approx 0-32: 0 < scaling_factor < 2^5 - * @min_sample_cycles: If the value of the GPU_ACTIVE counter (the number of - * cycles the GPU was working) is less than - * min_sample_cycles, the counter model will return an - * error, causing the IPA framework to approximate using - * the cached simple model results instead. This may be - * more accurate than extrapolating using a very small - * counter dump. - */ -struct kbase_ipa_model_vinstr_data { - struct kbase_device *kbdev; - s32 group_values[KBASE_IPA_MAX_GROUP_DEF_NUM]; - const struct kbase_ipa_group *groups_def; - size_t groups_def_num; - kbase_ipa_get_active_cycles_callback *get_active_cycles; - struct kbase_hwcnt_virtualizer_client *hvirt_cli; - struct kbase_hwcnt_dump_buffer dump_buf; - s32 reference_voltage; - s32 scaling_factor; - s32 min_sample_cycles; -}; - -/** - * struct kbase_ipa_group - represents a single IPA group - * @name: name of the IPA group - * @default_value: default value of coefficient for IPA group. - * Coefficients are interpreted as fractions where the - * denominator is 1000000. - * @op: which operation to be performed on the counter values - * @counter_block_offset: block offset in bytes of the counter used to calculate energy for IPA group - */ -struct kbase_ipa_group { - const char *name; - s32 default_value; - s64 (*op)(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter_block_offset); - u32 counter_block_offset; -}; - -/** - * kbase_ipa_sum_all_shader_cores() - sum a counter over all cores - * @model_data: pointer to model data - * @coeff: model coefficient. Unity is ~2^20, so range approx - * +/- 4.0: -2^22 < coeff < 2^22 - * @counter: offset in bytes of the counter used to calculate energy - * for IPA group - * - * Calculate energy estimation based on hardware counter `counter' - * across all shader cores. - * - * Return: Sum of counter values. Range: -2^54 < ret < 2^54 - */ -s64 kbase_ipa_sum_all_shader_cores(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter); - -/** - * kbase_ipa_sum_all_memsys_blocks() - sum a counter over all mem system blocks - * @model_data: pointer to model data - * @coeff: model coefficient. Unity is ~2^20, so range approx - * +/- 4.0: -2^22 < coeff < 2^22 - * @counter: offset in bytes of the counter used to calculate energy - * for IPA group - * - * Calculate energy estimation based on hardware counter `counter' across all - * memory system blocks. - * - * Return: Sum of counter values. Range: -2^51 < ret < 2^51 - */ -s64 kbase_ipa_sum_all_memsys_blocks(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter); - -/** - * kbase_ipa_single_counter() - sum a single counter - * @model_data: pointer to model data - * @coeff: model coefficient. Unity is ~2^20, so range approx - * +/- 4.0: -2^22 < coeff < 2^22 - * @counter: offset in bytes of the counter used to calculate energy - * for IPA group - * - * Calculate energy estimation based on hardware counter `counter'. - * - * Return: Counter value. Range: -2^49 < ret < 2^49 - */ -s64 kbase_ipa_single_counter(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter); - -/** - * kbase_ipa_attach_vinstr() - attach a vinstr_buffer to an IPA model. - * @model_data: pointer to model data - * - * Attach a vinstr_buffer to an IPA model. The vinstr_buffer - * allows access to the hardware counters used to calculate - * energy consumption. - * - * Return: 0 on success, or an error code. - */ -int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); - -/** - * kbase_ipa_detach_vinstr() - detach a vinstr_buffer from an IPA model. - * @model_data: pointer to model data - * - * Detach a vinstr_buffer from an IPA model. - */ -void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); - -/** - * kbase_ipa_vinstr_dynamic_coeff() - calculate dynamic power based on HW counters - * @model: pointer to instantiated model - * @coeffp: pointer to location where calculated power, in - * pW/(Hz V^2), is stored. - * - * This is a GPU-agnostic implementation of the get_dynamic_coeff() - * function of an IPA model. It relies on the model being populated - * with GPU-specific attributes at initialization time. - * - * Return: 0 on success, or an error code. - */ -int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp); - -/** - * kbase_ipa_vinstr_reset_data() - Reset the counters data used for dynamic - * power estimation - * @model: pointer to instantiated model - * - * Currently it is not implemented for JM GPUs. - * When implemented it is expected to retrieve the accumulated value of HW - * counters from the Vinstr component, without doing any processing, which is - * effectively a reset as the next call to kbase_ipa_counter_dynamic_coeff() - * will see the increment in counter values from this point onwards. - */ -void kbase_ipa_vinstr_reset_data(struct kbase_ipa_model *model); - -/** - * kbase_ipa_vinstr_common_model_init() - initialize ipa power model - * @model: ipa power model to initialize - * @ipa_groups_def: array of ipa groups which sets coefficients for - * the corresponding counters used in the ipa model - * @ipa_group_size: number of elements in the array @ipa_groups_def - * @get_active_cycles: callback to return the number of cycles the GPU was - * active during the counter sample period. - * @reference_voltage: voltage, in mV, of the operating point used when - * deriving the power model coefficients. - * - * This initialization function performs initialization steps common - * for ipa models based on counter values. In each call, the model - * passes its specific coefficient values per ipa counter group via - * @ipa_groups_def array. - * - * Return: 0 on success, error code otherwise - */ -int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, - const struct kbase_ipa_group *ipa_groups_def, - size_t ipa_group_size, - kbase_ipa_get_active_cycles_callback *get_active_cycles, - s32 reference_voltage); - -/** - * kbase_ipa_vinstr_common_model_term() - terminate ipa power model - * @model: ipa power model to terminate - * - * This function performs all necessary steps to terminate ipa power model - * including clean up of resources allocated to hold model data. - */ -void kbase_ipa_vinstr_common_model_term(struct kbase_ipa_model *model); - -#endif /* _KBASE_IPA_COUNTER_COMMON_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_jm.c b/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_jm.c deleted file mode 100644 index b19608a71b54..000000000000 --- a/drivers/gpu/arm/valhall/ipa/backend/mali_kbase_ipa_counter_jm.c +++ /dev/null @@ -1,536 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2016-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include - -#include "mali_kbase_ipa_counter_common_jm.h" -#include "mali_kbase.h" -#include - -/* Performance counter blocks base offsets */ -#define JM_BASE (0 * KBASE_IPA_NR_BYTES_PER_BLOCK) -#define MEMSYS_BASE (2 * KBASE_IPA_NR_BYTES_PER_BLOCK) - -/* JM counter block offsets */ -#define JM_GPU_ACTIVE (KBASE_IPA_NR_BYTES_PER_CNT * 6) - -/* MEMSYS counter block offsets */ -#define MEMSYS_L2_ANY_LOOKUP (KBASE_IPA_NR_BYTES_PER_CNT * 25) - -/* SC counter block offsets */ -#define SC_EXEC_INSTR_FMA (KBASE_IPA_NR_BYTES_PER_CNT * 27) -#define SC_EXEC_INSTR_COUNT (KBASE_IPA_NR_BYTES_PER_CNT * 28) -#define SC_EXEC_INSTR_MSG (KBASE_IPA_NR_BYTES_PER_CNT * 30) -#define SC_TEX_FILT_NUM_OPERATIONS (KBASE_IPA_NR_BYTES_PER_CNT * 39) -#define SC_TEX_COORD_ISSUE (KBASE_IPA_NR_BYTES_PER_CNT * 40) -#define SC_TEX_TFCH_NUM_OPERATIONS (KBASE_IPA_NR_BYTES_PER_CNT * 42) -#define SC_VARY_INSTR (KBASE_IPA_NR_BYTES_PER_CNT * 49) -#define SC_BEATS_WR_TIB (KBASE_IPA_NR_BYTES_PER_CNT * 62) - -/** - * kbase_g7x_power_model_get_jm_counter() - get performance counter offset - * inside the Job Manager block - * @model_data: pointer to GPU model data. - * @counter_block_offset: offset in bytes of the performance counter inside - * the Job Manager block. - * - * Return: Block offset in bytes of the required performance counter. - */ -static u32 kbase_g7x_power_model_get_jm_counter(struct kbase_ipa_model_vinstr_data *model_data, - u32 counter_block_offset) -{ - CSTD_UNUSED(model_data); - return JM_BASE + counter_block_offset; -} - -/** - * kbase_g7x_power_model_get_memsys_counter() - get performance counter offset - * inside the Memory System block - * @model_data: pointer to GPU model data. - * @counter_block_offset: offset in bytes of the performance counter inside - * the (first) Memory System block. - * - * Return: Block offset in bytes of the required performance counter. - */ -static u32 kbase_g7x_power_model_get_memsys_counter(struct kbase_ipa_model_vinstr_data *model_data, - u32 counter_block_offset) -{ - CSTD_UNUSED(model_data); - /* The base address of Memory System performance counters is always the same, although their number - * may vary based on the number of cores. For the moment it's ok to return a constant. - */ - return MEMSYS_BASE + counter_block_offset; -} - -/** - * kbase_g7x_power_model_get_sc_counter() - get performance counter offset - * inside the Shader Cores block - * @model_data: pointer to GPU model data. - * @counter_block_offset: offset in bytes of the performance counter inside - * the (first) Shader Cores block. - * - * Return: Block offset in bytes of the required performance counter. - */ -static u32 kbase_g7x_power_model_get_sc_counter(struct kbase_ipa_model_vinstr_data *model_data, - u32 counter_block_offset) -{ -#if IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) - const u32 sc_base = - MEMSYS_BASE + (KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS * KBASE_IPA_NR_BYTES_PER_BLOCK); -#else - const u32 sc_base = MEMSYS_BASE + (model_data->kbdev->gpu_props.num_l2_slices * - KBASE_IPA_NR_BYTES_PER_BLOCK); -#endif - return sc_base + counter_block_offset; -} - -/** - * kbase_g7x_sum_all_memsys_blocks() - calculate energy for a single Memory - * System performance counter. - * @model_data: pointer to GPU model data. - * @coeff: default value of coefficient for IPA group. - * @counter_block_offset: offset in bytes of the counter inside the block it - * belongs to. - * - * Return: Energy estimation for a single Memory System performance counter. - */ -static s64 kbase_g7x_sum_all_memsys_blocks(struct kbase_ipa_model_vinstr_data *model_data, - s32 coeff, u32 counter_block_offset) -{ - u32 counter; - - counter = kbase_g7x_power_model_get_memsys_counter(model_data, counter_block_offset); - return kbase_ipa_sum_all_memsys_blocks(model_data, coeff, counter); -} - -/** - * kbase_g7x_sum_all_shader_cores() - calculate energy for a Shader Cores - * performance counter for all cores. - * @model_data: pointer to GPU model data. - * @coeff: default value of coefficient for IPA group. - * @counter_block_offset: offset in bytes of the counter inside the block it - * belongs to. - * - * Return: Energy estimation for a Shader Cores performance counter for all - * cores. - */ -static s64 kbase_g7x_sum_all_shader_cores(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter_block_offset) -{ - u32 counter; - - counter = kbase_g7x_power_model_get_sc_counter(model_data, counter_block_offset); - return kbase_ipa_sum_all_shader_cores(model_data, coeff, counter); -} - -/** - * kbase_g7x_jm_single_counter() - calculate energy for a single Job Manager performance counter. - * @model_data: pointer to GPU model data. - * @coeff: default value of coefficient for IPA group. - * @counter_block_offset: offset in bytes of the counter inside the block it belongs to. - * - * Return: Energy estimation for a single Job Manager performance counter. - */ -static s64 kbase_g7x_jm_single_counter(struct kbase_ipa_model_vinstr_data *model_data, s32 coeff, - u32 counter_block_offset) -{ - u32 counter; - - counter = kbase_g7x_power_model_get_jm_counter(model_data, counter_block_offset); - return kbase_ipa_single_counter(model_data, coeff, counter); -} - -/** - * kbase_g7x_get_active_cycles() - return the GPU_ACTIVE counter - * @model_data: pointer to GPU model data. - * - * Return: the number of cycles the GPU was active during the counter sampling - * period. - */ -static u32 kbase_g7x_get_active_cycles(struct kbase_ipa_model_vinstr_data *model_data) -{ - u32 counter = kbase_g7x_power_model_get_jm_counter(model_data, JM_GPU_ACTIVE); - - /* Counters are only 32-bit, so we can safely multiply by 1 then cast - * the 64-bit result back to a u32. - */ - return kbase_ipa_single_counter(model_data, 1, counter); -} - -/* Table of IPA group definitions. - * - * For each IPA group, this table defines a function to access the given performance block counter (or counters, - * if the operation needs to be iterated on multiple blocks) and calculate energy estimation. - */ - -static const struct kbase_ipa_group ipa_groups_def_g71[] = { - { - .name = "l2_access", - .default_value = 526300, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, - { - .name = "exec_instr_count", - .default_value = 301100, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_COUNT, - }, - { - .name = "tex_issue", - .default_value = 197400, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_COORD_ISSUE, - }, - { - .name = "tile_wb", - .default_value = -156400, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_BEATS_WR_TIB, - }, - { - .name = "gpu_active", - .default_value = 115800, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_g72[] = { - { - .name = "l2_access", - .default_value = 393000, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, - { - .name = "exec_instr_count", - .default_value = 227000, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_COUNT, - }, - { - .name = "tex_issue", - .default_value = 181900, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_COORD_ISSUE, - }, - { - .name = "tile_wb", - .default_value = -120200, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_BEATS_WR_TIB, - }, - { - .name = "gpu_active", - .default_value = 133100, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_g76[] = { - { - .name = "gpu_active", - .default_value = 122000, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, - { - .name = "exec_instr_count", - .default_value = 488900, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_COUNT, - }, - { - .name = "vary_instr", - .default_value = 212100, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_VARY_INSTR, - }, - { - .name = "tex_tfch_num_operations", - .default_value = 288000, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, - }, - { - .name = "l2_access", - .default_value = 378100, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_g52_r1[] = { - { - .name = "gpu_active", - .default_value = 224200, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, - { - .name = "exec_instr_count", - .default_value = 384700, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_COUNT, - }, - { - .name = "vary_instr", - .default_value = 271900, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_VARY_INSTR, - }, - { - .name = "tex_tfch_num_operations", - .default_value = 477700, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, - }, - { - .name = "l2_access", - .default_value = 551400, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_g51[] = { - { - .name = "gpu_active", - .default_value = 201400, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, - { - .name = "exec_instr_count", - .default_value = 392700, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_COUNT, - }, - { - .name = "vary_instr", - .default_value = 274000, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_VARY_INSTR, - }, - { - .name = "tex_tfch_num_operations", - .default_value = 528000, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_TFCH_NUM_OPERATIONS, - }, - { - .name = "l2_access", - .default_value = 506400, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_g77[] = { - { - .name = "l2_access", - .default_value = 710800, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, - { - .name = "exec_instr_msg", - .default_value = 2375300, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_MSG, - }, - { - .name = "exec_instr_fma", - .default_value = 656100, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_FMA, - }, - { - .name = "tex_filt_num_operations", - .default_value = 318800, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, - }, - { - .name = "gpu_active", - .default_value = 172800, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_tbex[] = { - { - .name = "l2_access", - .default_value = 599800, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, - { - .name = "exec_instr_msg", - .default_value = 1830200, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_MSG, - }, - { - .name = "exec_instr_fma", - .default_value = 407300, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_FMA, - }, - { - .name = "tex_filt_num_operations", - .default_value = 224500, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, - }, - { - .name = "gpu_active", - .default_value = 153800, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, -}; - -static const struct kbase_ipa_group ipa_groups_def_tbax[] = { - { - .name = "l2_access", - .default_value = 599800, - .op = kbase_g7x_sum_all_memsys_blocks, - .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, - }, - { - .name = "exec_instr_msg", - .default_value = 1830200, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_MSG, - }, - { - .name = "exec_instr_fma", - .default_value = 407300, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_EXEC_INSTR_FMA, - }, - { - .name = "tex_filt_num_operations", - .default_value = 224500, - .op = kbase_g7x_sum_all_shader_cores, - .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, - }, - { - .name = "gpu_active", - .default_value = 153800, - .op = kbase_g7x_jm_single_counter, - .counter_block_offset = JM_GPU_ACTIVE, - }, -}; - -#define IPA_POWER_MODEL_OPS(gpu, init_token) \ - static const struct kbase_ipa_model_ops kbase_##gpu##_ipa_model_ops = { \ - .name = "mali-" #gpu "-power-model", \ - .init = kbase_##init_token##_power_model_init, \ - .term = kbase_ipa_vinstr_common_model_term, \ - .get_dynamic_coeff = kbase_ipa_vinstr_dynamic_coeff, \ - .reset_counter_data = kbase_ipa_vinstr_reset_data, \ - } - -#define STANDARD_POWER_MODEL(gpu, reference_voltage) \ - static int kbase_##gpu##_power_model_init(struct kbase_ipa_model *model) \ - { \ - BUILD_BUG_ON(ARRAY_SIZE(ipa_groups_def_##gpu) > KBASE_IPA_MAX_GROUP_DEF_NUM); \ - return kbase_ipa_vinstr_common_model_init(model, ipa_groups_def_##gpu, \ - ARRAY_SIZE(ipa_groups_def_##gpu), \ - kbase_g7x_get_active_cycles, \ - (reference_voltage)); \ - } \ - IPA_POWER_MODEL_OPS(gpu, gpu) - -#define ALIAS_POWER_MODEL(gpu, as_gpu) IPA_POWER_MODEL_OPS(gpu, as_gpu) - -STANDARD_POWER_MODEL(g71, 800); -STANDARD_POWER_MODEL(g72, 800); -STANDARD_POWER_MODEL(g76, 800); -STANDARD_POWER_MODEL(g52_r1, 1000); -STANDARD_POWER_MODEL(g51, 1000); -STANDARD_POWER_MODEL(g77, 1000); -STANDARD_POWER_MODEL(tbex, 1000); -STANDARD_POWER_MODEL(tbax, 1000); - -/* g52 is an alias of g76 (TNOX) for IPA */ -ALIAS_POWER_MODEL(g52, g76); -/* tnax is an alias of g77 (TTRX) for IPA */ -ALIAS_POWER_MODEL(tnax, g77); - -static const struct kbase_ipa_model_ops *ipa_counter_model_ops[] = { - &kbase_g71_ipa_model_ops, &kbase_g72_ipa_model_ops, &kbase_g76_ipa_model_ops, - &kbase_g52_ipa_model_ops, &kbase_g52_r1_ipa_model_ops, &kbase_g51_ipa_model_ops, - &kbase_g77_ipa_model_ops, &kbase_tnax_ipa_model_ops, &kbase_tbex_ipa_model_ops, - &kbase_tbax_ipa_model_ops -}; - -const struct kbase_ipa_model_ops *kbase_ipa_counter_model_ops_find(struct kbase_device *kbdev, - const char *name) -{ - size_t i; - - for (i = 0; i < ARRAY_SIZE(ipa_counter_model_ops); ++i) { - const struct kbase_ipa_model_ops *ops = ipa_counter_model_ops[i]; - - if (!strcmp(ops->name, name)) - return ops; - } - - dev_err(kbdev->dev, "power model \'%s\' not found\n", name); - - return NULL; -} - -const char *kbase_ipa_counter_model_name_from_id(struct kbase_gpu_id_props *gpu_id) -{ - switch (gpu_id->product_model) { - case GPU_ID_PRODUCT_TMIX: - return "mali-g71-power-model"; - case GPU_ID_PRODUCT_THEX: - return "mali-g72-power-model"; - case GPU_ID_PRODUCT_TNOX: - return "mali-g76-power-model"; - case GPU_ID_PRODUCT_TSIX: - return "mali-g51-power-model"; - case GPU_ID_PRODUCT_TGOX: - if (gpu_id->version_major == 0) - /* g52 aliased to g76 power-model's ops */ - return "mali-g52-power-model"; - else - return "mali-g52_r1-power-model"; - case GPU_ID_PRODUCT_TNAX: - return "mali-tnax-power-model"; - case GPU_ID_PRODUCT_TTRX: - return "mali-g77-power-model"; - case GPU_ID_PRODUCT_TBEX: - return "mali-tbex-power-model"; - case GPU_ID_PRODUCT_TBAX: - return "mali-tbax-power-model"; - default: - return NULL; - } -} diff --git a/drivers/gpu/arm/valhall/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/valhall/ipa/mali_kbase_ipa_simple.c index d8a356c3e8c2..b2dce96f4922 100644 --- a/drivers/gpu/arm/valhall/ipa/mali_kbase_ipa_simple.c +++ b/drivers/gpu/arm/valhall/ipa/mali_kbase_ipa_simple.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2016-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2016-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -34,8 +34,6 @@ #include "mali_kbase_ipa_simple.h" #include "mali_kbase_ipa_debugfs.h" -#if MALI_USE_CSF - /* This is used if the dynamic power for top-level is estimated separately * through the counter model. To roughly match the contribution of top-level * power in the total dynamic power, when calculated through counter model, @@ -46,8 +44,6 @@ */ #define TOP_LEVEL_DYN_COEFF_SCALER (3) -#endif /* MALI_USE_CSF */ - #if MALI_UNIT_TEST static int dummy_temp; @@ -208,7 +204,6 @@ static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) struct kbase_ipa_model_simple_data *model_data = (struct kbase_ipa_model_simple_data *)model->model_data; -#if MALI_USE_CSF /* On CSF GPUs, the dynamic power for top-level and shader cores is * estimated separately. Currently there is a single dynamic * coefficient value provided in the device tree for simple model. @@ -219,9 +214,6 @@ static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) coeffp[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL] = model_data->dynamic_coefficient / TOP_LEVEL_DYN_COEFF_SCALER; coeffp[KBASE_IPA_BLOCK_TYPE_SHADER_CORES] = model_data->dynamic_coefficient; -#else - *coeffp = model_data->dynamic_coefficient; -#endif return 0; } diff --git a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_defs.h b/drivers/gpu/arm/valhall/jm/mali_kbase_jm_defs.h deleted file mode 100644 index e9b388fabbda..000000000000 --- a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_defs.h +++ /dev/null @@ -1,770 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Definitions (types, defines, etcs) specific to Job Manager Kbase. - * They are placed here to allow the hierarchy of header files to work. - */ - -#ifndef _KBASE_JM_DEFS_H_ -#define _KBASE_JM_DEFS_H_ - -#include "mali_kbase_js_defs.h" - -#include - -/* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */ -#define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1 - -/* - * Number of milliseconds before resetting the GPU when a job cannot be "zapped" - * from the hardware. Note that the time is actually - * ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and - * the GPU actually being reset to give other contexts time for their jobs - * to be soft-stopped and removed from the hardware before resetting. - */ -#define ZAP_TIMEOUT 1000 - -/* - * Prevent soft-stops from occurring in scheduling situations - * - * This is not due to HW issues, but when scheduling is desired to be more - * predictable. - * - * Therefore, soft stop may still be disabled due to HW issues. - * - * Soft stop will still be used for non-scheduling purposes e.g. when - * terminating a context. - * - * if not in use, define this value to 0 instead of being undefined. - */ -#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0 - -/* - * Prevent hard-stops from occurring in scheduling situations - * - * This is not due to HW issues, but when scheduling is desired to be more - * predictable. - * - * Hard stop will still be used for non-scheduling purposes e.g. when - * terminating a context. - * - * if not in use, define this value to 0 instead of being undefined. - */ -#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 - -/* Atom has been previously soft-stopped */ -#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1U << 1) -/* Atom has been previously retried to execute */ -#define KBASE_KATOM_FLAGS_RERUN (1U << 2) -/* Atom submitted with JOB_CHAIN_FLAG bit set in JS_CONFIG_NEXT register, helps - * to disambiguate short-running job chains during soft/hard stopping of jobs - */ -#define KBASE_KATOM_FLAGS_JOBCHAIN (1U << 3) -/* Atom has been previously hard-stopped. */ -#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1U << 4) -/* Atom has caused us to enter disjoint state */ -#define KBASE_KATOM_FLAG_IN_DISJOINT (1U << 5) -/* Atom blocked on cross-slot dependency */ -#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1U << 7) -/* Atom has fail dependency on cross-slot dependency */ -#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1U << 8) -/* Atom is currently in the list of atoms blocked on cross-slot dependencies */ -#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1U << 9) -/* Atom requires GPU to be in protected mode */ -#define KBASE_KATOM_FLAG_PROTECTED (1U << 11) -/* Atom has been stored in runnable_tree */ -#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1U << 12) -/* Atom is waiting for L2 caches to power up in order to enter protected mode */ -#define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1U << 13) - -/* SW related flags about types of JS_COMMAND action - * NOTE: These must be masked off by JS_COMMAND_MASK - */ - -/* This command causes a disjoint event */ -#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100 - -/* Bitmask of all SW related flags */ -#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT) - -#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK) -#error "JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK." \ - "Must update JS_COMMAND_SW_<..> bitmasks" -#endif - -/* Soft-stop command that causes a Disjoint event. This of course isn't - * entirely masked off by JS_COMMAND_MASK - */ -#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP) - -#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT - -/* Serialize atoms within a slot (ie only one atom per job slot) */ -#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0) -/* Serialize atoms between slots (ie only one job slot running at any time) */ -#define KBASE_SERIALIZE_INTER_SLOT (1 << 1) -/* Reset the GPU after each atom completion */ -#define KBASE_SERIALIZE_RESET (1 << 2) - -/** - * enum kbase_timeout_selector - The choice of which timeout to get scaled - * using the lowest GPU frequency. - * @MMU_AS_INACTIVE_WAIT_TIMEOUT: Maximum waiting time in ms for the completion - * of a MMU operation - * @JM_DEFAULT_JS_FREE_TIMEOUT: Maximum timeout to wait for JS_COMMAND_NEXT - * to be updated on HW side so a Job Slot is - * considered free. - * @KBASE_PRFCNT_ACTIVE_TIMEOUT: Waiting time for prfcnt to be ready. - * @KBASE_CLEAN_CACHE_TIMEOUT: Waiting time for cache flush to complete. - * @KBASE_AS_INACTIVE_TIMEOUT: Waiting time for MCU address space to become inactive. - * @KBASE_TIMEOUT_SELECTOR_COUNT: Number of timeout selectors. - * @KBASE_DEFAULT_TIMEOUT: Fallthrough in case an invalid timeout is - * passed. - */ -enum kbase_timeout_selector { - MMU_AS_INACTIVE_WAIT_TIMEOUT, - JM_DEFAULT_JS_FREE_TIMEOUT, - KBASE_PRFCNT_ACTIVE_TIMEOUT, - KBASE_CLEAN_CACHE_TIMEOUT, - KBASE_AS_INACTIVE_TIMEOUT, - /* Must be the last in the enum */ - KBASE_TIMEOUT_SELECTOR_COUNT, - KBASE_DEFAULT_TIMEOUT = JM_DEFAULT_JS_FREE_TIMEOUT -}; - -#if IS_ENABLED(CONFIG_DEBUG_FS) -/** - * struct base_job_fault_event - keeps track of the atom which faulted or which - * completed after the faulty atom but before the - * debug data for faulty atom was dumped. - * - * @event_code: event code for the atom, should != BASE_JD_EVENT_DONE for - * the atom which faulted. - * @katom: pointer to the atom for which job fault occurred or which - * completed after the faulty atom. - * @job_fault_work: work item, queued only for the faulty atom, which waits for - * the dumping to get completed and then does the bottom half - * of job done for the atoms which followed the faulty atom. - * @head: List head used to store the atom in the global list of - * faulty atoms or context specific list of atoms which got - * completed during the dump. - * @reg_offset: offset of the register to be dumped next, only applicable - * for the faulty atom. - */ -struct base_job_fault_event { - u32 event_code; - struct kbase_jd_atom *katom; - struct work_struct job_fault_work; - struct list_head head; - int reg_offset; -}; -#endif - -/** - * struct kbase_jd_atom_dependency - Contains the dependency info for an atom. - * @atom: pointer to the dependee atom. - * @dep_type: type of dependency on the dependee @atom, i.e. order or data - * dependency. BASE_JD_DEP_TYPE_INVALID indicates no dependency. - */ -struct kbase_jd_atom_dependency { - struct kbase_jd_atom *atom; - u8 dep_type; -}; - -/** - * kbase_jd_katom_dep_atom - Retrieves a read-only reference to the - * dependee atom. - * @dep: pointer to the dependency info structure. - * - * Return: readonly reference to dependee atom. - */ -static inline const struct kbase_jd_atom * -kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep) -{ - return (const struct kbase_jd_atom *)(dep->atom); -} - -/** - * kbase_jd_katom_dep_type - Retrieves the dependency type info - * - * @dep: pointer to the dependency info structure. - * - * Return: the type of dependency there is on the dependee atom. - */ -static inline u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency *dep) -{ - return dep->dep_type; -} - -/** - * kbase_jd_katom_dep_set - sets up the dependency info structure - * as per the values passed. - * @const_dep: pointer to the dependency info structure to be setup. - * @a: pointer to the dependee atom. - * @type: type of dependency there is on the dependee atom. - */ -static inline void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency *const_dep, - struct kbase_jd_atom *a, u8 type) -{ - struct kbase_jd_atom_dependency *dep; - - dep = (struct kbase_jd_atom_dependency *)const_dep; - - dep->atom = a; - dep->dep_type = type; -} - -/** - * kbase_jd_katom_dep_clear - resets the dependency info structure - * - * @const_dep: pointer to the dependency info structure to be setup. - */ -static inline void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency *const_dep) -{ - struct kbase_jd_atom_dependency *dep; - - dep = (struct kbase_jd_atom_dependency *)const_dep; - - dep->atom = NULL; - dep->dep_type = BASE_JD_DEP_TYPE_INVALID; -} - -/** - * enum kbase_atom_gpu_rb_state - The state of an atom, pertinent after it - * becomes runnable, with respect to job slot - * ringbuffer/fifo. - * @KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: Atom not currently present in slot fifo, - * which implies that either atom has not become - * runnable due to dependency or has completed - * the execution on GPU. - * @KBASE_ATOM_GPU_RB_WAITING_BLOCKED: Atom has been added to slot fifo but is - * blocked due to cross slot dependency, - * can't be submitted to GPU. - * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: Atom has been added to slot - * fifo but is waiting for the completion of - * previously added atoms in current & other - * slots, as their protected mode requirements - * do not match with the current atom. - * @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: Atom is in slot fifo - * and is waiting for completion of protected - * mode transition, needed before the atom is - * submitted to GPU. - * @KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: Atom is in slot fifo but is - * waiting for the cores, which are needed to - * execute the job chain represented by the atom, - * to become available - * @KBASE_ATOM_GPU_RB_READY: Atom is in slot fifo and can be submitted to - * GPU. - * @KBASE_ATOM_GPU_RB_SUBMITTED: Atom is in slot fifo and has been submitted - * to GPU. - * @KBASE_ATOM_GPU_RB_RETURN_TO_JS: Atom must be returned to JS due to some - * failure, but only after the previously added - * atoms in fifo have completed or have also - * been returned to JS. - */ -enum kbase_atom_gpu_rb_state { - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB, - KBASE_ATOM_GPU_RB_WAITING_BLOCKED, - KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV, - KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION, - KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE, - KBASE_ATOM_GPU_RB_READY, - KBASE_ATOM_GPU_RB_SUBMITTED, - KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1 -}; - -/** - * enum kbase_atom_enter_protected_state - The state of an atom with respect to - * the preparation for GPU's entry into protected mode, - * becomes pertinent only after atom's state with respect - * to slot ringbuffer is - * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION - * @KBASE_ATOM_ENTER_PROTECTED_CHECK: Starting state. Check if there are any - * atoms currently submitted to GPU and protected mode - * transition is not already in progress. - * @KBASE_ATOM_ENTER_PROTECTED_HWCNT: Wait for hardware counter context to - * become disabled before entry into protected mode. - * @KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: Wait for the L2 to become idle in - * preparation for the coherency change. L2 shall be - * powered down and GPU shall come out of fully - * coherent mode before entering protected mode. - * @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change; - * for KBASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on - * so that coherency register contains correct value when - * GPU enters protected mode. - * @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for - * KBASE_HW_ISSUE_TGOX_R1_1234 check - * that L2 is powered up and switch GPU to protected mode. - */ -enum kbase_atom_enter_protected_state { - /* - * NOTE: The integer value of this must match - * KBASE_ATOM_EXIT_PROTECTED_CHECK. - */ - KBASE_ATOM_ENTER_PROTECTED_CHECK = 0, - KBASE_ATOM_ENTER_PROTECTED_HWCNT, - KBASE_ATOM_ENTER_PROTECTED_IDLE_L2, - KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY, - KBASE_ATOM_ENTER_PROTECTED_FINISHED, -}; - -/** - * enum kbase_atom_exit_protected_state - The state of an atom with respect to - * the preparation for GPU's exit from protected mode, - * becomes pertinent only after atom's state with respect - * to slot ngbuffer is - * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION - * @KBASE_ATOM_EXIT_PROTECTED_CHECK: Starting state. Check if there are any - * atoms currently submitted to GPU and protected mode - * transition is not already in progress. - * @KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: Wait for the L2 to become idle in - * preparation for the reset, as exiting protected mode - * requires a reset. - * @KBASE_ATOM_EXIT_PROTECTED_RESET: Issue the reset to trigger exit from - * protected mode - * @KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: End state, Wait for the reset to - * complete - */ -enum kbase_atom_exit_protected_state { - /* - * NOTE: The integer value of this must match - * KBASE_ATOM_ENTER_PROTECTED_CHECK. - */ - KBASE_ATOM_EXIT_PROTECTED_CHECK = 0, - KBASE_ATOM_EXIT_PROTECTED_IDLE_L2, - KBASE_ATOM_EXIT_PROTECTED_RESET, - KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT, -}; - -/** - * struct kbase_jd_atom - object representing the atom, containing the complete - * state and attributes of an atom. - * @work: work item for the bottom half processing of the atom, - * by JD or JS, after it got executed on GPU or the - * input fence got signaled - * @start_timestamp: time at which the atom was submitted to the GPU, by - * updating the JS_HEAD_NEXTn register. - * @udata: copy of the user data sent for the atom in - * base_jd_submit. - * @kctx: Pointer to the base context with which the atom is - * associated. - * @dep_head: Array of 2 list heads, pointing to the two list of - * atoms - * which are blocked due to dependency on this atom. - * @dep_item: Array of 2 list heads, used to store the atom in the - * list of other atoms depending on the same dependee - * atom. - * @dep: Array containing the dependency info for the 2 atoms - * on which the atom depends upon. - * @jd_item: List head used during job dispatch job_done - * processing - as dependencies may not be entirely - * resolved at this point, - * we need to use a separate list head. - * @in_jd_list: flag set to true if atom's @jd_item is currently on - * a list, prevents atom being processed twice. - * @jit_ids: Zero-terminated array of IDs of just-in-time memory - * allocations written to by the atom. When the atom - * completes, the value stored at the - * &struct_base_jit_alloc_info.heap_info_gpu_addr of - * each allocation is read in order to enforce an - * overall physical memory usage limit. - * @nr_extres: number of external resources referenced by the atom. - * @extres: Pointer to @nr_extres VA regions containing the external - * resource allocation and other information. - * @nr_extres external resources referenced by the atom. - * @device_nr: indicates the coregroup with which the atom is - * associated, when - * BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified. - * @jc: GPU address of the job-chain. - * @softjob_data: Copy of data read from the user space buffer that @jc - * points to. - * @dma_fence: object containing pointers to both input & output - * fences and other related members used for explicit - * sync through soft jobs and for the implicit - * synchronization required on access to external - * resources. - * @dma_fence.fence_in: Points to the dma-buf input fence for this atom. - * The atom would complete only after the fence is - * signaled. - * @dma_fence.fence: Points to the dma-buf output fence for this atom. - * @dma_fence.fence_cb: The object that is passed at the time of adding the - * callback that gets invoked when @dma_fence.fence_in - * is signaled. - * @dma_fence.fence_cb_added: Flag to keep a track if the callback was successfully - * added for @dma_fence.fence_in, which is supposed to be - * invoked on the signaling of fence. - * @dma_fence.context: The dma-buf fence context number for this atom. A - * unique context number is allocated to each katom in - * the context on context creation. - * @dma_fence.seqno: The dma-buf fence sequence number for this atom. This - * is increased every time this katom uses dma-buf fence - * @event_code: Event code for the job chain represented by the atom, - * both HW and low-level SW events are represented by - * event codes. - * @core_req: bitmask of BASE_JD_REQ_* flags specifying either - * Hw or Sw requirements for the job chain represented - * by the atom. - * @ticks: Number of scheduling ticks for which atom has been - * running on the GPU. - * @sched_priority: Priority of the atom for Job scheduling, as per the - * KBASE_JS_ATOM_SCHED_PRIO_*. - * @completed: Wait queue to wait upon for the completion of atom. - * @status: Indicates at high level at what stage the atom is in, - * as per KBASE_JD_ATOM_STATE_*, that whether it is not - * in use or its queued in JD or given to JS or - * submitted to Hw or it completed the execution on Hw. - * @slot_nr: Job slot chosen for the atom. - * @atom_flags: bitmask of KBASE_KATOM_FLAG* flags capturing the - * excat low level state of the atom. - * @gpu_rb_state: bitmnask of KBASE_ATOM_GPU_RB_* flags, precisely - * tracking atom's state after it has entered - * Job scheduler on becoming runnable. Atom - * could be blocked due to cross slot dependency - * or waiting for the shader cores to become available - * or waiting for protected mode transitions to - * complete. - * @need_cache_flush_cores_retained: flag indicating that manual flush of GPU - * cache is needed for the atom and the shader cores - * used for atom have been kept on. - * @blocked: flag indicating that atom's resubmission to GPU is - * blocked till the work item is scheduled to return the - * atom to JS. - * @seq_nr: user-space sequence number, to order atoms in some - * temporal order - * @pre_dep: Pointer to atom that this atom has same-slot - * dependency on - * @post_dep: Pointer to atom that has same-slot dependency on - * this atom - * @x_pre_dep: Pointer to atom that this atom has cross-slot - * dependency on - * @x_post_dep: Pointer to atom that has cross-slot dependency on - * this atom - * @flush_id: The GPU's flush count recorded at the time of - * submission, - * used for the cache flush optimization - * @fault_event: Info for dumping the debug data on Job fault. - * @queue: List head used for 4 different purposes : - * Adds atom to the list of dma-buf fence waiting atoms. - * Adds atom to the list of atoms blocked due to cross - * slot dependency. - * Adds atom to the list of softjob atoms for which JIT - * allocation has been deferred - * Adds atom to the list of softjob atoms waiting for - * the signaling of fence. - * @jit_node: Used to keep track of all JIT free/alloc jobs in - * submission order - * @jit_blocked: Flag indicating that JIT allocation requested through - * softjob atom will be reattempted after the impending - * free of other active JIT allocations. - * @will_fail_event_code: If non-zero, this indicates that the atom will fail - * with the set event_code when the atom is processed. - * Used for special handling of atoms, which have a data - * dependency on the failed atoms. - * @protected_state: State of the atom, as per - * KBASE_ATOM_(ENTER|EXIT)_PROTECTED_*, - * when transitioning into or out of protected mode. - * Atom will be either entering or exiting the - * protected mode. - * @protected_state.enter: entering the protected mode. - * @protected_state.exit: exiting the protected mode. - * @runnable_tree_node: The node added to context's job slot specific rb tree - * when the atom becomes runnable. - * @age: Age of atom relative to other atoms in the context, - * is snapshot of the age_count counter in kbase - * context. - * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. - */ -struct kbase_jd_atom { - struct work_struct work; - ktime_t start_timestamp; - - struct base_jd_udata udata; - struct kbase_context *kctx; - - struct list_head dep_head[2]; - struct list_head dep_item[2]; - const struct kbase_jd_atom_dependency dep[2]; - struct list_head jd_item; - bool in_jd_list; - -#if MALI_JIT_PRESSURE_LIMIT_BASE - u8 jit_ids[2]; -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - u16 nr_extres; - struct kbase_va_region **extres; - - u32 device_nr; - u64 jc; - void *softjob_data; -#if IS_ENABLED(CONFIG_SYNC_FILE) - struct { - /* Use the functions/API defined in mali_kbase_fence.h to - * when working with this sub struct - */ - struct dma_fence *fence_in; - /* This points to the dma-buf output fence for this atom. If - * this is NULL then there is no fence for this atom and the - * following fields related to dma_fence may have invalid data. - * - * The context and seqno fields contain the details for this - * fence. - * - * This fence is signaled when the katom is completed, - * regardless of the event_code of the katom (signal also on - * failure). - */ - struct dma_fence *fence; - - /* This is the callback object that is registered for the fence_in. - * The callback is invoked when the fence_in is signaled. - */ - struct dma_fence_cb fence_cb; - bool fence_cb_added; - - unsigned int context; - atomic_t seqno; - } dma_fence; -#endif /* CONFIG_SYNC_FILE */ - - /* Note: refer to kbasep_js_atom_retained_state, which will take a copy - * of some of the following members - */ - enum base_jd_event_code event_code; - base_jd_core_req core_req; - u8 jobslot; - - u32 ticks; - int sched_priority; - - wait_queue_head_t completed; - enum kbase_jd_atom_state status; - unsigned int slot_nr; - - u32 atom_flags; - - enum kbase_atom_gpu_rb_state gpu_rb_state; - - bool need_cache_flush_cores_retained; - - atomic_t blocked; - - u64 seq_nr; - - struct kbase_jd_atom *pre_dep; - struct kbase_jd_atom *post_dep; - - struct kbase_jd_atom *x_pre_dep; - struct kbase_jd_atom *x_post_dep; - - u32 flush_id; - -#if IS_ENABLED(CONFIG_DEBUG_FS) - struct base_job_fault_event fault_event; -#endif - struct list_head queue; - - struct list_head jit_node; - bool jit_blocked; - - enum base_jd_event_code will_fail_event_code; - - union { - enum kbase_atom_enter_protected_state enter; - enum kbase_atom_exit_protected_state exit; - } protected_state; - - struct rb_node runnable_tree_node; - - u32 age; -}; - -static inline bool kbase_jd_katom_is_protected(const struct kbase_jd_atom *katom) -{ - return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED); -} - -/** - * kbase_jd_atom_is_younger - query if one atom is younger by age than another - * - * @katom_a: the first atom - * @katom_b: the second atom - * - * Return: true if the first atom is strictly younger than the second, - * false otherwise. - */ -static inline bool kbase_jd_atom_is_younger(const struct kbase_jd_atom *katom_a, - const struct kbase_jd_atom *katom_b) -{ - return ((s32)(katom_a->age - katom_b->age) < 0); -} - -/** - * kbase_jd_atom_is_earlier - Check whether the first atom has been submitted - * earlier than the second one - * - * @katom_a: the first atom - * @katom_b: the second atom - * - * Return: true if the first atom has been submitted earlier than the - * second atom. It is used to understand if an atom that is ready has been - * submitted earlier than the currently running atom, so that the currently - * running atom should be preempted to allow the ready atom to run. - */ -static inline bool kbase_jd_atom_is_earlier(const struct kbase_jd_atom *katom_a, - const struct kbase_jd_atom *katom_b) -{ - /* No seq_nr set? */ - if (!katom_a->seq_nr || !katom_b->seq_nr) - return false; - - /* Efficiently handle the unlikely case of wrapping. - * The following code assumes that the delta between the sequence number - * of the two atoms is less than INT64_MAX. - * In the extremely unlikely case where the delta is higher, the comparison - * defaults for no preemption. - * The code also assumes that the conversion from unsigned to signed types - * works because the signed integers are 2's complement. - */ - return (s64)(katom_a->seq_nr - katom_b->seq_nr) < 0; -} - -/* - * Theory of operations: - * - * Atom objects are statically allocated within the context structure. - * - * Each atom is the head of two lists, one for the "left" set of dependencies, - * one for the "right" set. - */ - -#define KBASE_JD_DEP_QUEUE_SIZE 256 - -/** - * struct kbase_jd_context - per context object encapsulating all the - * Job dispatcher related state. - * @lock: lock to serialize the updates made to the - * Job dispatcher state and kbase_jd_atom objects. - * @sched_info: Structure encapsulating all the Job scheduling - * info. - * @atoms: Array of the objects representing atoms, - * containing the complete state and attributes - * of an atom. - * @job_nr: Tracks the number of atoms being processed by the - * kbase. This includes atoms that are not tracked by - * scheduler: 'not ready to run' & 'dependency-only' - * jobs. - * @zero_jobs_wait: Waitq that reflects whether there are no jobs - * (including SW-only dependency jobs). This is set - * when no jobs are present on the ctx, and clear - * when there are jobs. - * This must be updated atomically with @job_nr. - * note: Job Dispatcher knows about more jobs than - * the Job Scheduler as it is unaware of jobs that - * are blocked on dependencies and SW-only dependency - * jobs. This waitq can be waited upon to find out - * when the context jobs are all done/cancelled - * (including those that might've been blocked - * on dependencies) - and so, whether it can be - * terminated. However, it should only be terminated - * once it is not present in the run-pool. - * Since the waitq is only set under @lock, - * the waiter should also briefly obtain and drop - * @lock to guarantee that the setter has completed - * its work on the kbase_context - * @job_done_wq: Workqueue to which the per atom work item is - * queued for bottom half processing when the - * atom completes - * execution on GPU or the input fence get signaled. - * @tb_lock: Lock to serialize the write access made to @tb to - * store the register access trace messages. - * @tb: Pointer to the Userspace accessible buffer storing - * the trace messages for register read/write - * accesses made by the Kbase. The buffer is filled - * in circular fashion. - * @tb_wrap_offset: Offset to the end location in the trace buffer, - * the write pointer is moved to the beginning on - * reaching this offset. - * @jit_atoms_head: A list of the just-in-time memory soft-jobs, both - * allocate & free, in submission order, protected - * by kbase_jd_context.lock. - * @jit_pending_alloc: A list of just-in-time memory allocation - * soft-jobs which will be reattempted after the - * impending free of other active allocations. - * @max_priority: Max priority level allowed for this context. - */ -struct kbase_jd_context { - struct mutex lock; - struct kbasep_js_kctx_info sched_info; - struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; - struct workqueue_struct *job_done_wq; - - wait_queue_head_t zero_jobs_wait; - spinlock_t tb_lock; - u32 *tb; - u32 job_nr; - size_t tb_wrap_offset; - - struct list_head jit_atoms_head; - struct list_head jit_pending_alloc; - int max_priority; -}; - -/** - * struct jsctx_queue - JS context atom queue - * @runnable_tree: Root of RB-tree containing currently runnable atoms on this - * job slot. - * @x_dep_head: Head item of the linked list of atoms blocked on cross-slot - * dependencies. Atoms on this list will be moved to the - * runnable_tree when the blocking atom completes. - * - * hwaccess_lock must be held when accessing this structure. - */ -struct jsctx_queue { - struct rb_root runnable_tree; - struct list_head x_dep_head; -}; - -/** - * struct kbase_as - Object representing an address space of GPU. - * @number: Index at which this address space structure is present - * in an array of address space structures embedded inside - * the &struct kbase_device. - * @pf_wq: Workqueue for processing work items related to - * Page fault and Bus fault handling. - * @work_pagefault: Work item for the Page fault handling. - * @work_busfault: Work item for the Bus fault handling. - * @pf_data: Data relating to Page fault. - * @bf_data: Data relating to Bus fault. - * @current_setup: Stores the MMU configuration for this address space. - */ -struct kbase_as { - unsigned int number; - struct workqueue_struct *pf_wq; - struct work_struct work_pagefault; - struct work_struct work_busfault; - struct kbase_fault pf_data; - struct kbase_fault bf_data; - struct kbase_mmu_setup current_setup; -}; - -#endif /* _KBASE_JM_DEFS_H_ */ diff --git a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_js.h b/drivers/gpu/arm/valhall/jm/mali_kbase_jm_js.h deleted file mode 100644 index 65b54c68d8c7..000000000000 --- a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_js.h +++ /dev/null @@ -1,1002 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Job Scheduler Interface. - * These interfaces are Internal to KBase. - */ - -#ifndef _KBASE_JM_JS_H_ -#define _KBASE_JM_JS_H_ - -#include "mali_kbase_js_ctx_attr.h" - -#define JS_MAX_RUNNING_JOBS 8 - -/** - * kbasep_js_devdata_init - Initialize the Job Scheduler - * @kbdev: The kbase_device to operate on - * - * The struct kbasep_js_device_data sub-structure of kbdev must be zero - * initialized before passing to the kbasep_js_devdata_init() function. This is - * to give efficient error path code. - * - * Return: 0 on success, error code otherwise. - */ -int kbasep_js_devdata_init(struct kbase_device *const kbdev); - -/** - * kbasep_js_devdata_halt - Halt the Job Scheduler. - * @kbdev: The kbase_device to operate on - * - * It is safe to call this on kbdev even if it the kbasep_js_device_data - * sub-structure was never initialized/failed initialization, to give efficient - * error-path code. - * - * For this to work, the struct kbasep_js_device_data sub-structure of kbdev - * must be zero initialized before passing to the kbasep_js_devdata_init() - * function. This is to give efficient error path code. - * - * It is a programming error to call this whilst there are still kbase_context - * structures registered with this scheduler. - * - */ -void kbasep_js_devdata_halt(struct kbase_device *kbdev); - -/** - * kbasep_js_devdata_term - Terminate the Job Scheduler - * @kbdev: The kbase_device to operate on - * - * It is safe to call this on kbdev even if it the kbasep_js_device_data - * sub-structure was never initialized/failed initialization, to give efficient - * error-path code. - * - * For this to work, the struct kbasep_js_device_data sub-structure of kbdev - * must be zero initialized before passing to the kbasep_js_devdata_init() - * function. This is to give efficient error path code. - * - * It is a programming error to call this whilst there are still kbase_context - * structures registered with this scheduler. - */ -void kbasep_js_devdata_term(struct kbase_device *kbdev); - -/** - * kbasep_js_kctx_init - Initialize the Scheduling Component of a - * struct kbase_context on the Job Scheduler. - * @kctx: The kbase_context to operate on - * - * This effectively registers a struct kbase_context with a Job Scheduler. - * - * It does not register any jobs owned by the struct kbase_context with - * the scheduler. Those must be separately registered by kbasep_js_add_job(). - * - * The struct kbase_context must be zero initialized before passing to the - * kbase_js_init() function. This is to give efficient error path code. - * - * Return: 0 on success, error code otherwise. - */ -int kbasep_js_kctx_init(struct kbase_context *const kctx); - -/** - * kbasep_js_kctx_term - Terminate the Scheduling Component of a - * struct kbase_context on the Job Scheduler - * @kctx: The kbase_context to operate on - * - * This effectively de-registers a struct kbase_context from its Job Scheduler - * - * It is safe to call this on a struct kbase_context that has never had or - * failed initialization of its jctx.sched_info member, to give efficient - * error-path code. - * - * For this to work, the struct kbase_context must be zero intitialized before - * passing to the kbase_js_init() function. - * - * It is a Programming Error to call this whilst there are still jobs - * registered with this context. - */ -void kbasep_js_kctx_term(struct kbase_context *kctx); - -/* kbase_jsctx_slot_prio_blocked_set - Set a context as being blocked for a job - * slot at and below a given priority level - * @kctx: The kbase_context - * @js: The job slot - * @sched_prio: The priority levels that the context is blocked at for @js (all - * priority levels at this level and below will be blocked) - * - * To preserve ordering and dependencies of atoms on soft-stopping (both within - * an between priority levels), a context must be marked as blocked for that - * atom's job slot, for all priority levels at or below the atom's priority. - * - * This must only be called due to an atom that was pulled from the context, - * otherwise there will be no way of unblocking the context when the atom is - * completed/unpulled. - * - * Atoms of higher priority might still be able to be pulled from the context - * on @js. This helps with starting a high priority atom as soon as possible. - */ -static inline void kbase_jsctx_slot_prio_blocked_set(struct kbase_context *kctx, unsigned int js, - int sched_prio) -{ - struct kbase_jsctx_slot_tracking *slot_tracking = &kctx->slot_tracking[js]; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - WARN(!slot_tracking->atoms_pulled_pri[sched_prio], - "When marking slot %u as blocked for priority %d on a kctx, no atoms were pulled - the slot cannot become unblocked", - js, sched_prio); - - slot_tracking->blocked |= ((kbase_js_prio_bitmap_t)1) << sched_prio; - KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev, JS_SLOT_PRIO_BLOCKED, kctx, NULL, 0, js, - (unsigned int)sched_prio); -} - -/* kbase_jsctx_atoms_pulled - Return number of atoms pulled on a context - * @kctx: The kbase_context - * - * Having atoms pulled indicates the context is not idle. - * - * Return: the number of atoms pulled on @kctx - */ -static inline int kbase_jsctx_atoms_pulled(struct kbase_context *kctx) -{ - return atomic_read(&kctx->atoms_pulled_all_slots); -} - -/** - * kbasep_js_add_job - Add a job chain to the Job Scheduler, - * and take necessary actions to - * schedule the context/run the job. - * @kctx: The kbase_context to operate on - * @atom: Atom to add - * - * This atomically does the following: - * * Update the numbers of jobs information - * * Add the job to the run pool if necessary (part of init_job) - * - * Once this is done, then an appropriate action is taken: - * * If the ctx is scheduled, it attempts to start the next job (which might be - * this added job) - * * Otherwise, and if this is the first job on the context, it enqueues it on - * the Policy Queue - * - * The Policy's Queue can be updated by this in the following ways: - * * In the above case that this is the first job on the context - * * If the context is high priority and the context is not scheduled, then it - * could cause the Policy to schedule out a low-priority context, allowing - * this context to be scheduled in. - * - * If the context is already scheduled on the RunPool, then adding a job to it - * is guaranteed not to update the Policy Queue. And so, the caller is - * guaranteed to not need to try scheduling a context from the Run Pool - it - * can safely assert that the result is false. - * - * It is a programming error to have more than U32_MAX jobs in flight at a time. - * - * The following locking conditions are made on the caller: - * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * * it must not hold hwaccess_lock (as this will be obtained internally) - * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be - * obtained internally) - * * it must not hold kbasep_jd_device_data::queue_mutex (again, it's used - * internally). - * - * Return: true indicates that the Policy Queue was updated, and so the - * caller will need to try scheduling a context onto the Run Pool, - * false indicates that no updates were made to the Policy Queue, - * so no further action is required from the caller. This is always returned - * when the context is currently scheduled. - */ -bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom); - -/** - * kbasep_js_remove_job - Remove a job chain from the Job Scheduler, - * except for its 'retained state'. - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * @atom: Atom to remove - * - * Completely removing a job requires several calls: - * * kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of - * the atom - * * kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler - * * kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the - * remaining state held as part of the job having been run. - * - * In the common case of atoms completing normally, this set of actions is more - * optimal for spinlock purposes than having kbasep_js_remove_job() handle all - * of the actions. - * - * In the case of canceling atoms, it is easier to call - * kbasep_js_remove_cancelled_job(), which handles all the necessary actions. - * - * It is a programming error to call this when: - * * a atom is not a job belonging to kctx. - * * a atom has already been removed from the Job Scheduler. - * * a atom is still in the runpool - * - * Do not use this for removing jobs being killed by kbase_jd_cancel() - use - * kbasep_js_remove_cancelled_job() instead. - * - * The following locking conditions are made on the caller: - * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * - */ -void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *atom); - -/** - * kbasep_js_remove_cancelled_job - Completely remove a job chain from the - * Job Scheduler, in the case - * where the job chain was cancelled. - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * @katom: Atom to remove - * - * This is a variant of kbasep_js_remove_job() that takes care of removing all - * of the retained state too. This is generally useful for cancelled atoms, - * which need not be handled in an optimal way. - * - * It is a programming error to call this when: - * * a atom is not a job belonging to kctx. - * * a atom has already been removed from the Job Scheduler. - * * a atom is still in the runpool: - * * it is not being killed with kbasep_jd_cancel() - * - * The following locking conditions are made on the caller: - * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * * it must not hold the hwaccess_lock, (as this will be obtained - * internally) - * * it must not hold kbasep_js_device_data::runpool_mutex (as this could be - * obtained internally) - * - * Return: true indicates that ctx attributes have changed and the caller - * should call kbase_js_sched_all() to try to run more jobs and - * false otherwise. - */ -bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *katom); - -/** - * kbasep_js_runpool_requeue_or_kill_ctx - Handling the requeuing/killing of a - * context that was evicted from the - * policy queue or runpool. - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * @has_pm_ref: tells whether to release Power Manager active reference - * - * This should be used whenever handing off a context that has been evicted - * from the policy queue or the runpool: - * * If the context is not dying and has jobs, it gets re-added to the policy - * queue - * * Otherwise, it is not added - * - * In addition, if the context is dying the jobs are killed asynchronously. - * - * In all cases, the Power Manager active reference is released - * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. - * has_pm_ref must be set to false whenever the context was not previously in - * the runpool and does not hold a Power Manager active refcount. Note that - * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an - * active refcount even though they weren't in the runpool. - * - * The following locking conditions are made on the caller: - * * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * * it must not hold kbasep_jd_device_data::queue_mutex (as this will be - * obtained internally) - */ -void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, - bool has_pm_ref); - -/** - * kbasep_js_runpool_release_ctx - Release a refcount of a context being busy, - * allowing it to be scheduled out. - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * - * When the refcount reaches zero and the context might be scheduled out - * (depending on whether the Scheduling Policy has deemed it so, or if it has - * run out of jobs). - * - * If the context does get scheduled out, then The following actions will be - * taken as part of deschduling a context: - * For the context being descheduled: - * * If the context is in the processing of dying (all the jobs are being - * removed from it), then descheduling also kills off any jobs remaining in the - * context. - * * If the context is not dying, and any jobs remain after descheduling the - * context then it is re-enqueued to the Policy's Queue. - * * Otherwise, the context is still known to the scheduler, but remains absent - * from the Policy Queue until a job is next added to it. - * * In all descheduling cases, the Power Manager active reference (obtained - * during kbasep_js_try_schedule_head_ctx()) is released - * (kbase_pm_context_idle()). - * - * Whilst the context is being descheduled, this also handles actions that - * cause more atoms to be run: - * * Attempt submitting atoms when the Context Attributes on the Runpool have - * changed. This is because the context being scheduled out could mean that - * there are more opportunities to run atoms. - * * Attempt submitting to a slot that was previously blocked due to affinity - * restrictions. This is usually only necessary when releasing a context - * happens as part of completing a previous job, but is harmless nonetheless. - * * Attempt scheduling in a new context (if one is available), and if - * necessary, running a job from that new context. - * - * Unlike retaining a context in the runpool, this function cannot be called - * from IRQ context. - * - * It is a programming error to call this on a kctx that is not currently - * scheduled, or that already has a zero refcount. - * - * The following locking conditions are made on the caller: - * * it must not hold the hwaccess_lock, because it will be used internally. - * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be - * obtained internally) - * * it must not hold the kbase_device::mmu_hw_mutex (as this will be - * obtained internally) - * * it must not hold kbasep_jd_device_data::queue_mutex (as this will be - * obtained internally) - * - */ -void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbasep_js_runpool_release_ctx_and_katom_retained_state - Variant of - * kbasep_js_runpool_release_ctx() that handles additional - * actions from completing an atom. - * - * @kbdev: KBase device - * @kctx: KBase context - * @katom_retained_state: Retained state from the atom - * - * This is usually called as part of completing an atom and releasing the - * refcount on the context held by the atom. - * - * Therefore, the extra actions carried out are part of handling actions queued - * on a completed atom, namely: - * * Releasing the atom's context attributes - * * Retrying the submission on a particular slot, because we couldn't submit - * on that slot from an IRQ handler. - * - * The locking conditions of this function are the same as those for - * kbasep_js_runpool_release_ctx() - */ -void kbasep_js_runpool_release_ctx_and_katom_retained_state( - struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state); - -/** - * kbasep_js_runpool_release_ctx_nolock - Variant of kbase_js_runpool_release_ctx() - * without locks - * @kbdev: KBase device - * @kctx: KBase context - * - * Variant of kbase_js_runpool_release_ctx() that assumes that - * kbasep_js_device_data::runpool_mutex and - * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not - * attempt to schedule new contexts. - */ -void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbasep_js_schedule_privileged_ctx - Schedule in a privileged context - * - * @kbdev: KBase device - * @kctx: KBase context - * - * This schedules a context in regardless of the context priority. - * If the runpool is full, a context will be forced out of the runpool and the - * function will wait for the new context to be scheduled in. - * The context will be kept scheduled in (and the corresponding address space - * reserved) until kbasep_js_release_privileged_ctx is called). - * - * The following locking conditions are made on the caller: - * * it must not hold the hwaccess_lock, because it will be used internally. - * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be - * obtained internally) - * * it must not hold the kbase_device::mmu_hw_mutex (as this will be - * obtained internally) - * * it must not hold kbasep_jd_device_data::queue_mutex (again, it's used - * internally). - * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will - * be used internally. - * - */ -void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbasep_js_release_privileged_ctx - Release a privileged context, - * allowing it to be scheduled out. - * @kbdev: KBase device - * @kctx: KBase context - * - * See kbasep_js_runpool_release_ctx for potential side effects. - * - * The following locking conditions are made on the caller: - * * it must not hold the hwaccess_lock, because it will be used internally. - * * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex. - * * it must not hold kbasep_js_device_data::runpool_mutex (as this will be - * obtained internally) - * * it must not hold the kbase_device::mmu_hw_mutex (as this will be - * obtained internally) - * - */ -void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbase_js_try_run_jobs - Try to submit the next job on each slot - * @kbdev: KBase device - * - * The following locks may be used: - * * kbasep_js_device_data::runpool_mutex - * * hwaccess_lock - */ -void kbase_js_try_run_jobs(struct kbase_device *kbdev); - -/** - * kbasep_js_suspend - Suspend the job scheduler during a Power Management - * Suspend event. - * @kbdev: KBase device - * - * Causes all contexts to be removed from the runpool, and prevents any - * contexts from (re)entering the runpool. - * - * This does not handle suspending the one privileged context: the caller must - * instead do this by suspending the GPU HW Counter Instrumentation. - * - * This will eventually cause all Power Management active references held by - * contexts on the runpool to be released, without running any more atoms. - * - * The caller must then wait for all Power Management active refcount to become - * zero before completing the suspend. - * - * The emptying mechanism may take some time to complete, since it can wait for - * jobs to complete naturally instead of forcing them to end quickly. However, - * this is bounded by the Job Scheduler's Job Timeouts. Hence, this - * function is guaranteed to complete in a finite time. - */ -void kbasep_js_suspend(struct kbase_device *kbdev); - -/** - * kbasep_js_resume - Resume the Job Scheduler after a Power Management - * Resume event. - * @kbdev: KBase device - * - * This restores the actions from kbasep_js_suspend(): - * * Schedules contexts back into the runpool - * * Resumes running atoms on the GPU - */ -void kbasep_js_resume(struct kbase_device *kbdev); - -/** - * kbase_js_dep_resolved_submit - Submit an atom to the job scheduler. - * - * @kctx: Context pointer - * @katom: Pointer to the atom to submit - * - * The atom is enqueued on the context's ringbuffer. The caller must have - * ensured that all dependencies can be represented in the ringbuffer. - * - * Caller must hold jctx->lock - * - * Return: true if the context requires to be enqueued, otherwise false. - */ -bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_atom *katom); - -/** - * kbase_js_pull - Pull an atom from a context in the job scheduler for - * execution. - * - * @kctx: Context to pull from - * @js: Job slot to pull from - * - * The atom will not be removed from the ringbuffer at this stage. - * - * The HW access lock must be held when calling this function. - * - * Return: a pointer to an atom, or NULL if there are no atoms for this - * slot that can be currently run. - */ -struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js); - -/** - * kbase_js_unpull - Return an atom to the job scheduler ringbuffer. - * - * @kctx: Context pointer - * @katom: Pointer to the atom to unpull - * - * An atom is 'unpulled' if execution is stopped but intended to be returned to - * later. The most common reason for this is that the atom has been - * soft-stopped. Another reason is if an end-of-renderpass atom completed - * but will need to be run again as part of the same renderpass. - * - * Note that if multiple atoms are to be 'unpulled', they must be returned in - * the reverse order to which they were originally pulled. It is a programming - * error to return atoms in any other order. - * - * The HW access lock must be held when calling this function. - * - */ -void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom); - -/** - * kbase_js_complete_atom_wq - Complete an atom from jd_done_worker(), - * removing it from the job - * scheduler ringbuffer. - * @kctx: Context pointer - * @katom: Pointer to the atom to complete - * - * If the atom failed then all dependee atoms marked for failure propagation - * will also fail. - * - * Return: true if the context is now idle (no jobs pulled) false otherwise. - */ -bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom *katom); - -/** - * kbase_js_complete_atom - Complete an atom. - * - * @katom: Pointer to the atom to complete - * @end_timestamp: The time that the atom completed (may be NULL) - * - * Most of the work required to complete an atom will be performed by - * jd_done_worker(). - * - * The HW access lock must be held when calling this function. - * - * Return: a atom that has now been unblocked and can now be run, or NULL - * if none - */ -struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp); - -/** - * kbase_js_sched - Submit atoms from all available contexts. - * - * @kbdev: Device pointer - * @js_mask: Mask of job slots to submit to - * - * This will attempt to submit as many jobs as possible to the provided job - * slots. It will exit when either all job slots are full, or all contexts have - * been used. - * - */ -void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask); - -/** - * kbase_js_zap_context - Attempt to deschedule a context that is being - * destroyed - * @kctx: Context pointer - * - * This will attempt to remove a context from any internal job scheduler queues - * and perform any other actions to ensure a context will not be submitted - * from. - * - * If the context is currently scheduled, then the caller must wait for all - * pending jobs to complete before taking any further action. - */ -void kbase_js_zap_context(struct kbase_context *kctx); - -/** - * kbase_js_is_atom_valid - Validate an atom - * - * @kbdev: Device pointer - * @katom: Atom to validate - * - * This will determine whether the atom can be scheduled onto the GPU. Atoms - * with invalid combinations of core requirements will be rejected. - * - * Return: true if atom is valid false otherwise. - */ -bool kbase_js_is_atom_valid(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -/** - * kbase_js_set_timeouts - update all JS timeouts with user specified data - * - * @kbdev: Device pointer - * - * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is - * set to a positive number then that becomes the new value used, if a timeout - * is negative then the default is set. - */ -void kbase_js_set_timeouts(struct kbase_device *kbdev); - -/** - * kbase_js_set_ctx_priority - set the context priority - * - * @kctx: Context pointer - * @new_priority: New priority value for the Context - * - * The context priority is set to a new value and it is moved to the - * pullable/unpullable list as per the new priority. - */ -void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority); - -/** - * kbase_js_update_ctx_priority - update the context priority - * - * @kctx: Context pointer - * - * The context priority gets updated as per the priority of atoms currently in - * use for that context, but only if system priority mode for context scheduling - * is being used. - */ -void kbase_js_update_ctx_priority(struct kbase_context *kctx); - -/* - * Helpers follow - */ - -/** - * kbasep_js_is_submit_allowed - Check that a context is allowed to submit - * jobs on this policy - * @js_devdata: KBase Job Scheduler Device Data - * @kctx: KBase context - * - * The purpose of this abstraction is to hide the underlying data size, - * and wrap up the long repeated line of code. - * - * As with any bool, never test the return value with true. - * - * The caller must hold hwaccess_lock. - * - * Return: true if the context is allowed to submit jobs, false otherwise. - */ -static inline bool kbasep_js_is_submit_allowed(struct kbasep_js_device_data *js_devdata, - struct kbase_context *kctx) -{ - u16 test_bit; - bool is_allowed; - - /* Ensure context really is scheduled in */ - if (WARN((kctx->as_nr == KBASEP_AS_NR_INVALID) || !kbase_ctx_flag(kctx, KCTX_SCHEDULED), - "%s: kctx %pK has assigned AS %d and context flag %d\n", __func__, (void *)kctx, - kctx->as_nr, atomic_read(&kctx->flags))) - return false; - - test_bit = (u16)(1u << kctx->as_nr); - - is_allowed = (bool)(js_devdata->runpool_irq.submit_allowed & test_bit); - dev_dbg(kctx->kbdev->dev, "JS: submit %s allowed on %pK (as=%d)", - is_allowed ? "is" : "isn't", (void *)kctx, kctx->as_nr); - return is_allowed; -} - -/** - * kbasep_js_set_submit_allowed - Allow a context to submit jobs on this policy - * @js_devdata: KBase Job Scheduler Device Data - * @kctx: KBase context - * - * The purpose of this abstraction is to hide the underlying data size, - * and wrap up the long repeated line of code. - * - * The caller must hold hwaccess_lock. - */ -static inline void kbasep_js_set_submit_allowed(struct kbasep_js_device_data *js_devdata, - struct kbase_context *kctx) -{ - u16 set_bit; - - /* Ensure context really is scheduled in */ - if (WARN((kctx->as_nr == KBASEP_AS_NR_INVALID) || !kbase_ctx_flag(kctx, KCTX_SCHEDULED), - "%s: kctx %pK has assigned AS %d and context flag %d\n", __func__, (void *)kctx, - kctx->as_nr, atomic_read(&kctx->flags))) - return; - - set_bit = (u16)(1u << kctx->as_nr); - - dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %pK (as=%d)", kctx, kctx->as_nr); - - js_devdata->runpool_irq.submit_allowed |= set_bit; -} - -/** - * kbasep_js_clear_submit_allowed - Prevent a context from submitting more - * jobs on this policy - * @js_devdata: KBase Job Scheduler Device Data - * @kctx: KBase context - * - * The purpose of this abstraction is to hide the underlying data size, - * and wrap up the long repeated line of code. - * - * The caller must hold hwaccess_lock. - */ -static inline void kbasep_js_clear_submit_allowed(struct kbasep_js_device_data *js_devdata, - struct kbase_context *kctx) -{ - u16 clear_bit; - u16 clear_mask; - - /* Ensure context really is scheduled in */ - if (WARN((kctx->as_nr == KBASEP_AS_NR_INVALID) || !kbase_ctx_flag(kctx, KCTX_SCHEDULED), - "%s: kctx %pK has assigned AS %d and context flag %d\n", __func__, (void *)kctx, - kctx->as_nr, atomic_read(&kctx->flags))) - return; - - clear_bit = (u16)(1u << kctx->as_nr); - clear_mask = ~clear_bit; - - dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %pK (as=%d)", kctx, kctx->as_nr); - - js_devdata->runpool_irq.submit_allowed &= clear_mask; -} - -/** - * kbasep_js_atom_retained_state_init_invalid - Create an initial 'invalid' - * atom retained state - * - * @retained_state: pointer where to create and initialize the state - * - * Create an initial 'invalid' atom retained state, that requires no - * atom-related work to be done on releasing with - * kbasep_js_runpool_release_ctx_and_katom_retained_state() - */ -static inline void -kbasep_js_atom_retained_state_init_invalid(struct kbasep_js_atom_retained_state *retained_state) -{ - retained_state->event_code = BASE_JD_EVENT_NOT_STARTED; - retained_state->core_req = KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID; -} - -/** - * kbasep_js_atom_retained_state_copy() - Copy atom state - * @retained_state: where to copy - * @katom: where to copy from - * - * Copy atom state that can be made available after kbase_jd_done_nolock() is called - * on that atom. - */ -static inline void -kbasep_js_atom_retained_state_copy(struct kbasep_js_atom_retained_state *retained_state, - const struct kbase_jd_atom *katom) -{ - retained_state->event_code = katom->event_code; - retained_state->core_req = katom->core_req; - retained_state->sched_priority = katom->sched_priority; - retained_state->device_nr = katom->device_nr; -} - -/** - * kbasep_js_has_atom_finished - Determine whether an atom has finished - * (given its retained state), - * and so should be given back to - * userspace/removed from the system. - * - * @katom_retained_state: the retained state of the atom to check - * - * Reasons for an atom not finishing include: - * * Being soft-stopped (and so, the atom should be resubmitted sometime later) - * * It is an end of renderpass atom that was run to consume the output of a - * start-of-renderpass atom that was soft-stopped because it used too much - * memory. In this case, it will have to be run again later. - * - * Return: false if the atom has not finished, true otherwise. - */ -static inline bool -kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state) -{ - return (bool)(katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && - katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT); -} - -/** - * kbasep_js_atom_retained_state_is_valid - Determine whether a struct - * kbasep_js_atom_retained_state - * is valid - * @katom_retained_state: the atom's retained state to check - * - * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates - * that the code should just ignore it. - * - * Return: false if the retained state is invalid, true otherwise. - */ -static inline bool kbasep_js_atom_retained_state_is_valid( - const struct kbasep_js_atom_retained_state *katom_retained_state) -{ - return (bool)(katom_retained_state->core_req != - KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID); -} - -/** - * kbase_js_runpool_inc_context_count - Increment number of running contexts. - * @kbdev: KBase device - * @kctx: KBase context - * - * The following locking conditions are made on the caller: - * * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. - * * The caller must hold the kbasep_js_device_data::runpool_mutex - */ -static inline void kbase_js_runpool_inc_context_count(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - lockdep_assert_held(&js_devdata->runpool_mutex); - - /* Track total contexts */ - WARN_ON_ONCE(js_devdata->nr_all_contexts_running >= JS_MAX_RUNNING_JOBS); - ++(js_devdata->nr_all_contexts_running); - - if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { - /* Track contexts that can submit jobs */ - WARN_ON_ONCE(js_devdata->nr_user_contexts_running >= JS_MAX_RUNNING_JOBS); - ++(js_devdata->nr_user_contexts_running); - } -} - -/** - * kbase_js_runpool_dec_context_count - decrement number of running contexts. - * - * @kbdev: KBase device - * @kctx: KBase context - * The following locking conditions are made on the caller: - * * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. - * * The caller must hold the kbasep_js_device_data::runpool_mutex - */ -static inline void kbase_js_runpool_dec_context_count(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - lockdep_assert_held(&js_devdata->runpool_mutex); - - /* Track total contexts */ - --(js_devdata->nr_all_contexts_running); - WARN_ON_ONCE(js_devdata->nr_all_contexts_running < 0); - - if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { - /* Track contexts that can submit jobs */ - --(js_devdata->nr_user_contexts_running); - WARN_ON_ONCE(js_devdata->nr_user_contexts_running < 0); - } -} - -/** - * kbase_js_sched_all - Submit atoms from all available contexts to all - * job slots. - * - * @kbdev: Device pointer - * - * This will attempt to submit as many jobs as possible. It will exit when - * either all job slots are full, or all contexts have been used. - */ -static inline void kbase_js_sched_all(struct kbase_device *kbdev) -{ - kbase_js_sched(kbdev, (1U << kbdev->gpu_props.num_job_slots) - 1U); -} - -extern const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS]; - -extern const base_jd_prio kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; - -/** - * kbasep_js_atom_prio_to_sched_prio - Convert atom priority (base_jd_prio) - * to relative ordering. - * @atom_prio: Priority ID to translate. - * - * Atom priority values for @ref base_jd_prio cannot be compared directly to - * find out which are higher or lower. - * - * This function will convert base_jd_prio values for successively lower - * priorities into a monotonically increasing sequence. That is, the lower the - * base_jd_prio priority, the higher the value produced by this function. This - * is in accordance with how the rest of the kernel treats priority. - * - * The mapping is 1:1 and the size of the valid input range is the same as the - * size of the valid output range, i.e. - * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS - * - * Note This must be kept in sync with BASE_JD_PRIO_<...> definitions - * - * Return: On success: a value in the inclusive range - * 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure: - * KBASE_JS_ATOM_SCHED_PRIO_INVALID - */ -static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio) -{ - if (atom_prio >= BASE_JD_NR_PRIO_LEVELS) - return KBASE_JS_ATOM_SCHED_PRIO_INVALID; - - return kbasep_js_atom_priority_to_relative[atom_prio]; -} - -/** - * kbasep_js_sched_prio_to_atom_prio - Convert relative scheduler priority - * to atom priority (base_jd_prio). - * - * @kbdev: Device pointer - * @sched_prio: Relative scheduler priority to translate. - * - * This function will convert relative scheduler priority back into base_jd_prio - * values. It takes values which priorities are monotonically increasing - * and converts them to the corresponding base_jd_prio values. If an invalid number is - * passed in (i.e. not within the expected range) an error code is returned instead. - * - * The mapping is 1:1 and the size of the valid input range is the same as the - * size of the valid output range, i.e. - * KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS - * - * Return: On success: a value in the inclusive range - * 0..BASE_JD_NR_PRIO_LEVELS-1. On failure: BASE_JD_PRIO_INVALID. - */ -static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(struct kbase_device *kbdev, - int sched_prio) -{ - if (likely(sched_prio >= 0 && sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT)) - return kbasep_js_relative_priority_to_atom[sched_prio]; - /* Invalid priority value if reached here */ - dev_warn(kbdev->dev, "Unknown JS scheduling priority %d", sched_prio); - return BASE_JD_PRIO_INVALID; -} - -/** - * kbase_js_priority_check - Check the priority requested - * - * @kbdev: Device pointer - * @priority: Requested priority - * - * This will determine whether the requested priority can be satisfied. - * - * Return: The same or lower priority than requested. - */ -base_jd_prio kbase_js_priority_check(struct kbase_device *kbdev, base_jd_prio priority); - -/** - * kbase_js_atom_runs_before - determine if atoms for the same slot have an - * ordering relation - * @kbdev: kbase device - * @katom_a: the first atom - * @katom_b: the second atom. - * @order_flags: combination of KBASE_ATOM_ORDERING_FLAG_<...> for the ordering - * relation - * - * This is for making consistent decisions about the ordering of atoms when we - * need to do pre-emption on a slot, which includes stopping existing atoms - * when a new atom is ready to run, and also which other atoms to remove from - * the slot when the atom in JS_HEAD is being pre-empted. - * - * This only handles @katom_a and @katom_b being for the same job slot, as - * pre-emption only operates within a slot. - * - * Note: there is currently no use-case for this as a sorting comparison - * functions, hence only a boolean returned instead of int -1, 0, +1 return. If - * required in future, a modification to do so would be better than calling - * twice with katom_a and katom_b swapped. - * - * Return: - * true if @katom_a should run before @katom_b, false otherwise. - * A false return value does not distinguish between "no ordering relation" and - * "@katom_a should run after @katom_b". - */ -bool kbase_js_atom_runs_before(struct kbase_device *kbdev, const struct kbase_jd_atom *katom_a, - const struct kbase_jd_atom *katom_b, - const kbase_atom_ordering_flag_t order_flags); - -#endif /* _KBASE_JM_JS_H_ */ diff --git a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h b/drivers/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h deleted file mode 100644 index 98767fc06527..000000000000 --- a/drivers/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h +++ /dev/null @@ -1,53 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_JM_MEM_FLAGS_H_ -#define _KBASE_JM_MEM_FLAGS_H_ - -#include - -/* Kernel-side only flags allocated from 63 bit downwards */ - -/* Use the GPU VA chosen by the kernel client */ -#define BASEP_MEM_FLAG_MAP_FIXED ((base_mem_alloc_flags)1 << 61) - -/* Force trimming of JIT allocations when creating a new allocation */ -#define BASEP_MEM_PERFORM_JIT_TRIM ((base_mem_alloc_flags)1 << 60) - -/* Region belongs to a shrinker. - * - * This can either mean that it is part of the JIT/Ephemeral or tiler heap - * shrinker paths. Should be removed only after making sure that there are - * no references remaining to it in these paths, as it may cause the physical - * backing of the region to disappear during use. - */ -#define BASEP_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 59) - -/* Allocation is actively used for JIT memory */ -#define BASEP_MEM_ACTIVE_JIT_ALLOC ((base_mem_alloc_flags)1 << 58) - -/* The first available bit which can be used to define new memory flag - * if needed. It should be decremented by one once new flag is added - * and BASEP_MEM_FLAGS_NR_BITS should be incremented accordingly - */ -#define BASEP_MEM_FIRST_FREE_FLAG ((base_mem_alloc_flags)1 << 57) - -#endif /* _KBASE_JM_MEM_FLAGS_H_ */ diff --git a/drivers/gpu/arm/valhall/jm/mali_kbase_js_defs.h b/drivers/gpu/arm/valhall/jm/mali_kbase_js_defs.h deleted file mode 100644 index c81af50918fd..000000000000 --- a/drivers/gpu/arm/valhall/jm/mali_kbase_js_defs.h +++ /dev/null @@ -1,466 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Job Scheduler Type Definitions - */ - -#ifndef _KBASE_JS_DEFS_H_ -#define _KBASE_JS_DEFS_H_ - -/* Forward decls */ -struct kbase_device; -struct kbase_jd_atom; - -typedef u32 kbase_context_flags; - -/* - * typedef kbasep_js_ctx_job_cb - Callback function run on all of a context's - * jobs registered with the Job Scheduler - */ -typedef void kbasep_js_ctx_job_cb(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -/* - * @brief Maximum number of jobs that can be submitted to a job slot whilst - * inside the IRQ handler. - * - * This is important because GPU NULL jobs can complete whilst the IRQ handler - * is running. Otherwise, it potentially allows an unlimited number of GPU NULL - * jobs to be submitted inside the IRQ handler, which increases IRQ latency. - */ -#define KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ 2 - -/** - * enum kbasep_js_ctx_attr - Context attributes - * @KBASEP_JS_CTX_ATTR_COMPUTE: Attribute indicating a context that contains - * Compute jobs. - * @KBASEP_JS_CTX_ATTR_NON_COMPUTE: Attribute indicating a context that contains - * Non-Compute jobs. - * @KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES: Attribute indicating that a context - * contains compute-job atoms that aren't - * restricted to a coherent group, - * and can run on all cores. - * @KBASEP_JS_CTX_ATTR_COUNT: Must be the last in the enum - * - * Each context attribute can be thought of as a boolean value that caches some - * state information about either the runpool, or the context: - * - In the case of the runpool, it is a cache of "Do any contexts owned by - * the runpool have attribute X?" - * - In the case of a context, it is a cache of "Do any atoms owned by the - * context have attribute X?" - * - * The boolean value of the context attributes often affect scheduling - * decisions, such as affinities to use and job slots to use. - * - * To accomodate changes of state in the context, each attribute is refcounted - * in the context, and in the runpool for all running contexts. Specifically: - * - The runpool holds a refcount of how many contexts in the runpool have this - * attribute. - * - The context holds a refcount of how many atoms have this attribute. - * - * KBASEP_JS_CTX_ATTR_COMPUTE: - * Attribute indicating a context that contains Compute jobs. That is, - * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE - * - * @note A context can be both 'Compute' and 'Non Compute' if it contains - * both types of jobs. - * - * KBASEP_JS_CTX_ATTR_NON_COMPUTE: - * Attribute indicating a context that contains Non-Compute jobs. That is, - * the context has some jobs that are \b not of type @ref - * BASE_JD_REQ_ONLY_COMPUTE. - * - * @note A context can be both 'Compute' and 'Non Compute' if it contains - * both types of jobs. - * - * KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES: - * Attribute indicating that a context contains compute-job atoms that - * aren't restricted to a coherent group, and can run on all cores. - * - * Specifically, this is when the atom's \a core_req satisfy: - * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 - * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups - * - * Such atoms could be blocked from running if one of the coherent groups - * is being used by another job slot, so tracking this context attribute - * allows us to prevent such situations. - * - * @note This doesn't take into account the 1-coregroup case, where all - * compute atoms would effectively be able to run on 'all cores', but - * contexts will still not always get marked with this attribute. Instead, - * it is the caller's responsibility to take into account the number of - * coregroups when interpreting this attribute. - * - * @note Whilst Tiler atoms are normally combined with - * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without - * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy - * enough to handle anyway. - * - */ -enum kbasep_js_ctx_attr { - KBASEP_JS_CTX_ATTR_COMPUTE, - KBASEP_JS_CTX_ATTR_NON_COMPUTE, - KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, - KBASEP_JS_CTX_ATTR_COUNT -}; - -enum { - /* - * Bit indicating that new atom should be started because this atom - * completed - */ - KBASE_JS_ATOM_DONE_START_NEW_ATOMS = (1u << 0), - /* - * Bit indicating that the atom was evicted from the JS_NEXT registers - */ - KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT = (1u << 1) -}; - -/** - * typedef kbasep_js_atom_done_code - Combination of KBASE_JS_ATOM_DONE_<...> - * bits - */ -typedef u32 kbasep_js_atom_done_code; - -/* - * Context scheduling mode defines for kbase_device::js_ctx_scheduling_mode - */ -enum { - /* - * In this mode, higher priority atoms will be scheduled first, - * regardless of the context they belong to. Newly-runnable higher - * priority atoms can preempt lower priority atoms currently running on - * the GPU, even if they belong to a different context. - */ - KBASE_JS_SYSTEM_PRIORITY_MODE = 0, - - /* - * In this mode, the highest-priority atom will be chosen from each - * context in turn using a round-robin algorithm, so priority only has - * an effect within the context an atom belongs to. Newly-runnable - * higher priority atoms can preempt the lower priority atoms currently - * running on the GPU, but only if they belong to the same context. - */ - KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE, - - /* Must be the last in the enum */ - KBASE_JS_PRIORITY_MODE_COUNT, -}; - -/* - * Internal atom priority defines for kbase_jd_atom::sched_prio - */ -enum { - KBASE_JS_ATOM_SCHED_PRIO_FIRST = 0, - KBASE_JS_ATOM_SCHED_PRIO_REALTIME = KBASE_JS_ATOM_SCHED_PRIO_FIRST, - KBASE_JS_ATOM_SCHED_PRIO_HIGH, - KBASE_JS_ATOM_SCHED_PRIO_MED, - KBASE_JS_ATOM_SCHED_PRIO_LOW, - KBASE_JS_ATOM_SCHED_PRIO_COUNT, -}; - -/* Invalid priority for kbase_jd_atom::sched_prio */ -#define KBASE_JS_ATOM_SCHED_PRIO_INVALID -1 - -/* Default priority in the case of contexts with no atoms, or being lenient - * about invalid priorities from userspace. - */ -#define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED - -/* Atom priority bitmaps, where bit 0 is the highest priority, and higher bits - * indicate successively lower KBASE_JS_ATOM_SCHED_PRIO_<...> levels. - * - * Must be strictly larger than the number of bits to represent a bitmap of - * priorities, so that we can do calculations such as: - * (1 << KBASE_JS_ATOM_SCHED_PRIO_COUNT) - 1 - * ...without causing undefined behavior due to a shift beyond the width of the - * type - * - * If KBASE_JS_ATOM_SCHED_PRIO_COUNT starts requiring 32 bits, then it's worth - * moving to DECLARE_BITMAP() - */ -typedef u8 kbase_js_prio_bitmap_t; - -/* Ordering modification for kbase_js_atom_runs_before() */ -typedef u32 kbase_atom_ordering_flag_t; - -/* Atoms of the same context and priority should have their ordering decided by - * their seq_nr instead of their age. - * - * seq_nr is used as a more slowly changing variant of age - it increases once - * per group of related atoms, as determined by user-space. Hence, it can be - * used to limit re-ordering decisions (such as pre-emption) to only re-order - * between such groups, rather than re-order within those groups of atoms. - */ -#define KBASE_ATOM_ORDERING_FLAG_SEQNR (((kbase_atom_ordering_flag_t)1) << 0) - -/** - * struct kbasep_js_device_data - KBase Device Data Job Scheduler sub-structure - * @runpool_irq: Sub-structure to collect together Job Scheduling data used in - * IRQ context. The hwaccess_lock must be held when accessing. - * @runpool_irq.submit_allowed: Bitvector indicating whether a currently - * scheduled context is allowed to submit jobs. - * When bit 'N' is set in this, it indicates whether - * the context bound to address space 'N' is - * allowed to submit jobs. - * @runpool_irq.ctx_attr_ref_count: Array of Context Attributes Ref_counters: - * Each is large enough to hold a refcount of the number of contexts - * that can fit into the runpool. This is currently BASE_MAX_NR_AS. - * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store - * the refcount. Hence, it's not worthwhile reducing this to - * bit-manipulation on u32s to save space (where in contrast, 4 bit - * sub-fields would be easy to do and would save space). - * Whilst this must not become negative, the sign bit is used for: - * - error detection in debug builds - * - Optimization: it is undefined for a signed int to overflow, and so - * the compiler can optimize for that never happening (thus, no masking - * is required on updating the variable) - * @runpool_irq.slot_affinities: Affinity management and tracking. Bitvector - * to aid affinity checking. - * Element 'n' bit 'i' indicates that slot 'n' - * is using core i (i.e. slot_affinity_refcount[n][i] > 0) - * @runpool_irq.slot_affinity_refcount: Array of fefcount for each core owned - * by each slot. Used to generate the slot_affinities array of bitvectors. - * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, - * because it is refcounted only when a job is definitely about to be - * submitted to a slot, and is de-refcounted immediately after a job - * finishes - * @schedule_sem: Scheduling semaphore. This must be held when calling - * kbase_jm_kick() - * @ctx_list_pullable: List of contexts that can currently be pulled from - * @ctx_list_unpullable: List of contexts that can not currently be pulled - * from, but have jobs currently running. - * @nr_user_contexts_running: Number of currently scheduled user contexts - * (excluding ones that are not submitting jobs) - * @nr_all_contexts_running: Number of currently scheduled contexts (including - * ones that are not submitting jobs) - * @js_reqs: Core Requirements to match up with base_js_atom's core_req memeber - * @note This is a write-once member, and so no locking is required to - * read - * @scheduling_period_ns: Value for JS_SCHEDULING_PERIOD_NS - * @soft_stop_ticks: Value for JS_SOFT_STOP_TICKS - * @soft_stop_ticks_cl: Value for JS_SOFT_STOP_TICKS_CL - * @hard_stop_ticks_ss: Value for JS_HARD_STOP_TICKS_SS - * @hard_stop_ticks_cl: Value for JS_HARD_STOP_TICKS_CL - * @hard_stop_ticks_dumping: Value for JS_HARD_STOP_TICKS_DUMPING - * @gpu_reset_ticks_ss: Value for JS_RESET_TICKS_SS - * @gpu_reset_ticks_cl: Value for JS_RESET_TICKS_CL - * @gpu_reset_ticks_dumping: Value for JS_RESET_TICKS_DUMPING - * @ctx_timeslice_ns: Value for JS_CTX_TIMESLICE_NS - * @suspended_soft_jobs_list: List of suspended soft jobs - * @softstop_always: Support soft-stop on a single context - * @init_status:The initialized-flag is placed at the end, to avoid - * cache-pollution (we should only be using this during init/term paths). - * @note This is a write-once member, and so no locking is required to - * read - * @nr_contexts_pullable:Number of contexts that can currently be pulled from - * @nr_contexts_runnable:Number of contexts that can either be pulled from or - * arecurrently running - * @soft_job_timeout_ms:Value for JS_SOFT_JOB_TIMEOUT - * @js_free_wait_time_ms: Maximum waiting time in ms for a Job Slot to be seen free. - * @queue_mutex: Queue Lock, used to access the Policy's queue of contexts - * independently of the Run Pool. - * Of course, you don't need the Run Pool lock to access this. - * @runpool_mutex: Run Pool mutex, for managing contexts within the runpool. - * - * This encapsulates the current context of the Job Scheduler on a particular - * device. This context is global to the device, and is not tied to any - * particular struct kbase_context running on the device. - * - * nr_contexts_running and as_free are optimized for packing together (by making - * them smaller types than u32). The operations on them should rarely involve - * masking. The use of signed types for arithmetic indicates to the compiler - * that the value will not rollover (which would be undefined behavior), and so - * under the Total License model, it is free to make optimizations based on - * that (i.e. to remove masking). - */ -struct kbasep_js_device_data { - struct runpool_irq { - u16 submit_allowed; - s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; - u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; - s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; - } runpool_irq; - struct semaphore schedule_sem; - struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; - struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; - s8 nr_user_contexts_running; - s8 nr_all_contexts_running; - base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; - - u32 scheduling_period_ns; - u32 soft_stop_ticks; - u32 soft_stop_ticks_cl; - u32 hard_stop_ticks_ss; - u32 hard_stop_ticks_cl; - u32 hard_stop_ticks_dumping; - u32 gpu_reset_ticks_ss; - u32 gpu_reset_ticks_cl; - u32 gpu_reset_ticks_dumping; - u32 ctx_timeslice_ns; - - struct list_head suspended_soft_jobs_list; - -#ifdef CONFIG_MALI_VALHALL_DEBUG - bool softstop_always; -#endif /* CONFIG_MALI_VALHALL_DEBUG */ - int init_status; - u32 nr_contexts_pullable; - atomic_t nr_contexts_runnable; - atomic_t soft_job_timeout_ms; - u32 js_free_wait_time_ms; - - struct mutex queue_mutex; - /* - * Run Pool mutex, for managing contexts within the runpool. - * Unless otherwise specified, you must hold this lock whilst accessing - * any members that follow - * - * In addition, this is used to access: - * * the kbasep_js_kctx_info::runpool substructure - */ - struct mutex runpool_mutex; - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - /** - * @gpu_metrics_timer: High-resolution timer used to periodically emit the GPU metrics - * tracepoints for applications that are using the GPU. The timer is - * needed for the long duration handling so that the length of work - * period is within the allowed limit. - */ - struct hrtimer gpu_metrics_timer; - - /** - * @gpu_metrics_timer_needed: Flag to indicate if the @gpu_metrics_timer is needed. - * The timer won't be started after the expiry if the flag - * isn't set. - */ - bool gpu_metrics_timer_needed; - - /** - * @gpu_metrics_timer_running: Flag to indicate if the @gpu_metrics_timer is running. - * The flag is set to false when the timer is cancelled or - * is not restarted after the expiry. - */ - bool gpu_metrics_timer_running; -#endif -}; - -/** - * struct kbasep_js_kctx_info - KBase Context Job Scheduling information - * structure - * @ctx: Job Scheduler Context information sub-structure.Its members are - * accessed regardless of whether the context is: - * - In the Policy's Run Pool - * - In the Policy's Queue - * - Not queued nor in the Run Pool. - * You must obtain the @ctx.jsctx_mutex before accessing any other members - * of this substructure. - * You may not access any of its members from IRQ context. - * @ctx.jsctx_mutex: Job Scheduler Context lock - * @ctx.nr_jobs: Number of jobs ready to run - does \em not include - * the jobs waiting in the dispatcher, and dependency-only - * jobs. See kbase_jd_context::job_nr for such jobs - * @ctx.ctx_attr_ref_count: Context Attributes ref count. Each is large enough - * to hold a refcount of the number of atoms on the context. - * @ctx.is_scheduled_wait: Wait queue to wait for KCTX_SHEDULED flag state - * changes. - * @ctx.ctx_list_entry: Link implementing JS queues. Context can be present on - * one list per job slot. - * @init_status: The initalized-flag is placed at the end, to avoid - * cache-pollution (we should only be using this during init/term paths) - * - * This is a substructure in the struct kbase_context that encapsulates all the - * scheduling information. - */ -struct kbasep_js_kctx_info { - struct kbase_jsctx { - struct mutex jsctx_mutex; - - u32 nr_jobs; - u32 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; - wait_queue_head_t is_scheduled_wait; - struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS]; - } ctx; - int init_status; -}; - -/** - * struct kbasep_js_atom_retained_state - Subset of atom state. - * @event_code: to determine whether the atom has finished - * @core_req: core requirements - * @sched_priority: priority - * @device_nr: Core group atom was executed on - * - * Subset of atom state that can be available after kbase_jd_done_nolock() is called - * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), - * because the original atom could disappear. - */ -struct kbasep_js_atom_retained_state { - /* Event code - to determine whether the atom has finished */ - enum base_jd_event_code event_code; - /* core requirements */ - base_jd_core_req core_req; - /* priority */ - int sched_priority; - /* Core group atom was executed on */ - u32 device_nr; -}; - -/* - * Value signifying 'no retry on a slot required' for: - * - kbase_js_atom_retained_state::retry_submit_on_slot - * - kbase_jd_atom::retry_submit_on_slot - */ -#define KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID (-1) - -/* - * base_jd_core_req value signifying 'invalid' for a - * kbase_jd_atom_retained_state. See kbase_atom_retained_state_is_valid() - */ -#define KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID BASE_JD_REQ_DEP - -/* - * The JS timer resolution, in microseconds - * Any non-zero difference in time will be at least this size. - */ -#define KBASEP_JS_TICK_RESOLUTION_US 1 - -/** - * struct kbase_jsctx_slot_tracking - Job Scheduling tracking of a context's - * use of a job slot - * @blocked: bitmap of priorities that this slot is blocked at - * @atoms_pulled: counts of atoms that have been pulled from this slot, - * across all priority levels - * @atoms_pulled_pri: counts of atoms that have been pulled from this slot, per - * priority level - * - * Controls how a slot from the &struct kbase_context's jsctx_queue is managed, - * for example to ensure correct ordering of atoms when atoms of different - * priorities are unpulled. - */ -struct kbase_jsctx_slot_tracking { - kbase_js_prio_bitmap_t blocked; - atomic_t atoms_pulled; - int atoms_pulled_pri[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; -}; - -#endif /* _KBASE_JS_DEFS_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_csffw.bin b/drivers/gpu/arm/valhall/mali_csffw.bin index d95a7e530576d67c5404c3de7f0bb09bec7460d0..5d6480e8c70f5511e940050c38f404428cf0e7c3 100644 GIT binary patch delta 21258 zcmZ{M30PA{*YM2E&Bh`@R6tN}SObCtS8%Ce@v3zJw-%Sy;2KdYF0Ifl!KG?9v7>La zwwA81+C`AI?t-mW+v?lIrL~J1yQ$FSCSr0)Ao;1j&;VOH}6VniC*t$7E z-Y_4dx~+TfMLwUva;W7LIggVtBERE=0u~h_6@&-~8VEfg(7%Oi9$mckT!~tCgLVxle$TC4j zUX=BfuLhKoG|FNJtJp%cXz}Vft4tNcxFG`v4VXK$Vvv5p(D{S%hYnp>F+@LP=z;}# zg9a7k&tFimVBye$L38I0B3-gvCJ@=wpB)*Ady>hT9^`}Q15I__bPM{j z!h#0RwLrgr{}s=*SZ`dyIkJK$gONP>b6K_M48c7T#;r=IMK2`QqWCIHu!2tm^No?h z#9kIOXO#sld&GibpvKP3TC^;?_OFI>0Y9q7g0?$f}Q|K|Mmh5?69E7ofb41!gL4|cUeen_nxx0UBM9!&Mi%=MN7jA(K-mHAk2cW z6GDFoA40ev0tZAag3ymGBoFp_Xh_^ahB1B2Fo!;3n8x&4bQ9XpK!*wl;Sl^#=PHC> zAPk1NjDzq{W>agg-fA=Dc?4YbB>dJ5fjI)_BUtUk7%5G;9w7_bHVT*?@+tGfALBU4 zNaeZUbXmkn-yK9*%cjZGHx8NjsnEoRioSdjW73+iMo z==5F-`a&MeK)(W9zTbi>6%zg(fHdt2r5ltM^!Yvu8kBEAHG3>*W(eL_Af?v>gbvUS zfVAF@;ZpiNfHb{yqy;(OHe1lyA`3eBjs>wLurvoO=y8BF-FLLq(2fv15Q01};n^_~ zoHABw_(g!UVKJ1RJ5Ea1g`n|%son1Y(t-H_E;mDk zfOKHnLTPUZ4l9-NXNTa&A$ThUlgp%fMt~o|1U~>s2arA4g3Pp|Ll9~Dw*|d4MapOa zNb#3KFaq`%O(%ySe-FaGqXWu*K!T5j;GPgX8-lF>=@LbhOYJ{1Lu#-8ELi{8_bg~` zn3Qn>Anhn_wgejAc;D>p zT%3gO50K(J*IUruk1c4nd9xHb5`rV1mD1PhENI3d3rgH-LGOQPL9YP(64K8Q5K$*A z=)`uZp&{{7!;@aKpmlEpSBK#IdI`T0Anni_N}qjGD#x6X+A(Xslt{w?q-#`v&VsHU zx1g&3NO<rAcJS663mSgR zf*xy^VE4Nc?DR+jeWa77Vf{Y|MxX+&4OaMARvTm%l^P`f5ewQKf)_$i5hmf|A|&`s z2wn<7MWlqE6oSpf?06nP?F~Zq%%s* zsSQd_iCF6kze zbdcTIUZvg3LB<{OTFzp5tgoB!Jc;Z-6!&Qw-Cv9GTCylt%UYB}FO%nUXGWgs`sFR^Oj#mhM$-f8n!LO3jgI>iiHq9Gci}A3g zUxqA_<84h7M+|51=BCZ}J%pJ)N1JYqT928C4)RE0`RG=!uG-)>P1g9ZalHGxy!mzl|`ESs30i-Yu+w+zgdN$43ZZKGxy)tn!yZ{3jA#lobAssN*N| zMZ88hKn53mo*;H{JZ?CCODK*MMjnE=R!XtLo~HO>8H3l5KBIRrnVsaw=t)fXuBI@) zn@ZjTG_UndeI_Xx{3J;$tHB?W*UIML2_#UqS-H8(`Kv<6AzLQDiaSZplr7A%PI6*O zGBc}_{5T~YA1CUmlkqk(YwA>7N%l`2Cr=z$==yl7tS?@S7{lO1a3L@;;UUwBmre;|n8*coFERrnbQPdk1mm~paG>4ynUw~YyG%;G?KQZj8)pPr zNF(g_?uER9P@dI$8&Yekq>^>s%q0ewCzRUkT@0zyh8A;hmDhT+mM&RpaJxe*^SF24 z(l$u<)JTyuC1L?R>18Ik7d$lKOP& zO9iB+<1G{Wl^I-uJ`^_^TtAbMGZM^J=-~d)fS&6Z3&+ErF_<^F!a|g&?J&4b7$=8H zJPsvPp%T+Ww5tM~M%yrk;SKH=xbRTP1%QnUVbA`Boe9{tLiM*%+EU7u0g?+L8$tx< z0aq{K+&vH-v_=>KxSvDizB5R;0f4iGaG!_j>Hv2sgj*LHWM9Dj5W+19O|1vu%#IK; zD^xiQkhg;4&ga8nX_yY!%&v+5MWqf=?ZoN{K>iu~!<{(%s8beYZqM3pa7Tp(W$H4x zP!EH91?*YG(t*x1CBx4BL7cZV88*ZRoHY1L;w7`l=3Nqz*5DR267mc2l2`&b1K?f? z;aWt4`wzX8Kdh_R#jyt0tg3sBYoY1=;_D`ZYiF5MW{-FSQg4Rl`MOxK*x)LulJZ^_ zZ$K(BlxHU5yvpQCgF7-brstr0M=yix$52!2AZ>~@xbj04o)BMO3Tzq5T@3?M8l;B* z=)@dO?Iyh12`4U;W;j%hbxQkqS2?uJaOAa&cC?T+<~N zmJ;WztN3g3)$CcyC?QESxKt$Xq2bs|mOXTx8Qw)EJ)Fwuy2$E>pJr46(*AIIWH+Iw zxKjk3$MtaeY32=Ju(5VZsj``8=cDC?4Lk zQGB{yEj&&#=Z<8U05Q!q;K#_`xeqZPbQ0A(4nIgT=dClZZNPQN%)1SFu$d#IJ&3fh zsUz^+xN1~~ca*qWWK1=b&;u2??mig;SstVI$-3R)6GVL!(jmcNxh9q)N29Vj!pXA3 znP19v{A^KX>@#CSQ3udfHF;hM)ogH^GWN+QXDlyu2Ug+ixaOVa;xEHbw#rzoW4Bn& zv?wF&*F;spwcD!4u89%qD#IK@ezD4q(h=9qaZQ}o{o3v4jn5wlb6mU4DzAxgMtt8u z!)I}0oCqN;N_FuwMb#+7z)x=@Big~}RLx<|*TZ*-$jJ8ZZ&UWqwMF#LwSOm~s%zre z{A=R5a`SHSy6T#EfQFPa9m5K0-VZ@G<%cZ-L|?iQa!NWd~y zh+<1xWyMHfSR_QC;#SsN422EFt;jGi$ua-R0kuwk!QfK%O!Fe4MTzZaqhyZq12f*7 zDsqThIMMB&hqc#4BQPYB(zn{!h^l#41Ov;(lr6$ug%D%H`CO|^-V5lto+VDArivK|dzCIc6zLMGQ>0uUnkF*a+xz7Asu{4yE2q#tv}Py9==$hIYu5});_uj%7RGt1IBK9e87 zqh{31Rm-&9gr{hs33wM7zjO>Uz)!XU{J-aoATu}N+`wE?MnNJ5}-%nCjW+Xh; zwH?;eAYilsppN%>D*Pyw16BSKS+FuEVcv3CG!YmJAK(tI+#YOFhB$!jl8$^8-5+;+URt>~P z(zq%f4~?kB#~*g0s0U!y^I_I{BN9$oC`F1_=S1}vLALBg$*iCM%PD)zTkzWnx2?jc9H9on<=TUSGy^+tuzA41QUXt`^2L6c87)=*$QE=V$*l2aHgQgpbKrj}A;& zaIY+AmZk@S<||7$Xcc!LgHmRreM}{3Yo>?q z_1_!SV`Rgce)u$LTr&~Zk-(bi38(T9oDhY0vdD2*FVnpwzN?oxv1~2y+w$rH>?3=t zpNPEF#c)7rSXd-HO!8_9@%v;|O$zX`wPtkh`@Q%+r8_Rx-2TXZw8QU7@zSLa7a{`0 zUDF@7NZRA?%Zou}J4L>Ed|6*X#Diry|FLttOjsmh4h*`DmXG|NQE-S^j+62yhQ;(c zdI;7GHhPxKX5gctS2ub4iGJ8Y>`!FibEM;mp>UGqJvj!yNot-Pj}MU3Pv$Bzd{vF= zH1fxWxHP3O{m4J}V5D%PD;+rR_^m2Kz_XVOeX1??RUgaWYNXuwwlRC5N?6mW<5j|k zq~Pf>_!g;sI*pAKo)*czrw!4$UWKEoZ-dt&6sp1c6t)oG(@WF`I~WI}RSLh6<>ndq zDe}2FEg^aU(zduPOv`PVmUgEx_&lS*VpR#}h}yCdPbOO}@py34yB0IXkCJ|CbD2RM z(C+ zjy$z)cH zUv2tb-KSH*)5iZKr#JtuH28jLG`My&K}ig3>v?VL=Wm0n8ddnYQH$6XqxV>#{)x&#J3nKItMS>IP)& z0o4VBQq?!2QT2^DKkmz5pmCGxO9)nRsp>g#DT&&e!knh--7B)ga|6gIF=PyRaO-Hi zn7q0*At9$z$?L&QGz>)Ou&(O(Fs)id8myNK{~{N+KC1Y&%j;1D$fTFDa0;n;={3Bh zDV{_z_!W}5Jq7m3#O;gl1LULaNwKf^aGH*zCI7IM6_7&00+E)X?7g8#1>ENM zEch>G=j@0ZV#S5HA8C1`yDZkhk%z3A1J{c!D$tG_gyRh#v3w+d7Gxv!9Vvuk9edzR zq z=_R^7s~M2ayZ4NZ^81_bly1@W33qp5kH^z=ch5!4%`TN8nr`k}g7JqOZwU)S449W)1Gt`>am9J~XirTVd(05ug zQ3eM7EmTK1W`J^N8KY~YI;^Etg}xw3hx*~~$oNBf!>@>}S1CMG{4ei20_qQv-Q%*e zJ6aUT`LPA3j(3}k&_9QHu%_}gLLf?2rD_g%v;pv;l8+8)6NFkBClfCDzPqWf);CIJ z^uaQiI1eR&P$u0D_hkw@NdLpVdc(QMox0r%QGvN9D0x}u9;6atz2xD;_u(bv)x!lN z*MeKLI@m1>s>;63VwWDQEU-5bHX@GE%7l?#x?=j~uA7sc{wG4Y17z@n(n#fJsdN(rA3$49J-$S8+W>JR!GQ z4jcEr5(Fj=K1OrK2}N)($|4}Jkz)I_%s94(Z~lUKhdvgE5HiWtS$ zq6~43dDXJaxMsP~=FKj8``pk8<1pCBBEKzDrfe zXCVHo9fmjrrNN-pAVXqdnCUQyGPw^mU^C7%25`~Xa{@A+0=P}=p++<<={LG-L zPsa=8gqmt)vVOK`_B_Pt^y5nyo`Qc3?I9ssUDrPBk>iesrQp zm(5E~=`+oH*=A(12OH9}iaLnA=zEhZ=(s2Lqo19m-G!)JjwOm_{{_rl(z% zp2=?&^@jP4=B;AO*vxE&eXDq7Y+~b9@!PRD8{5uR)`Ovm-Eq-vc09CCi7M9t-uZun z-uCA&Ey~>B?xK4OnuVQZDX(kA@=@(K-lvQQ43`OmR&DpbEx-UCz^QcK083;-Y28kd zL5yxx`?>d(K83}b>3PXdx7dbaS{&ujqQBdZ{{oEwFH=6Mt&$1%dz*eCOOIuNYu#=NH+Z20z9tM5$i4zrhr#NuIL*+t5HCJ_S(K8L?Mtv1+ zj`}^K?S-yW9L+~2U1Oo^EQxH+1g&vsa~hMbYLDqg!UH-cTgHzyp3(^^cv~@6j+BL!^WWoZ*wM7P@ z&$?ol4wtqgy}`vsfL*(2qPuTY=yl(=x%!XSR`HEaqCsW^U0q85E`2X|2= z98TOh@f__q>t&Dx80I*lcD!4IO9cxhs#aC5GWtA!_)snq@@jck3r3uD)5Q%a6xndn zaEi@Ky#c3es#=%;3L!Omkuc8tw>N6Yj||5^?#RD#hml`SElJMp1d<`F4R|6%*lR&4 z%LOIqToG|mn}Yz!H;(+(-|Gx3=T7B zFcX^9BYhJUDoeWLppde{NKt-x(_s$GV5#9eg!SviJhTIrrbXGe`H;v0W)J*ILWqUV zslXW3zGc6ovPD$Y9|o5M>yGPTv%}6s$`WP=?xt7i%AfG#q9O&IsEtL85`U@ zNoSLA85wmpDaPWhK0MKV%!q?K7grU!&XE;oW0Y#)F>o4M$O}+%CVBttsC4iRa#s@j znF?JI^4bH^egRQZ0sEo?52HMF;efiD#C<*_Z$yAf6_DhHFGV4W(twgyVno(1&7KjG zjt65~Iy}!3^XHTCd}9B672Zike{qmm0FIR}jD2V8N0!iC39b_Z$9q6MR`5Ew<{7}N zg&2`M`Q>h~)f3Kr+3TTz!5uOANQp8z%&DLo8LNb;0n&MH7&ACP^3T78edud>U*t!YnGHKv}y*Q+&x|4P&jaV7#EooK|0Qaz!}AkW@TNAIo`eANQt1Nn*ugSA^t304_nJi z-oKiQe{8yXwG=bMz<|46#Kif@%Iih3?%*=1Ldz5rq?=M`BVS%G2IV!%u|_Qv(uS16 zX0pXGnpxOIs(&nEO1j9gAC28(g0;mY+953y4x0Q=L%>n^>5T<{RRl8pI2vy!Ykn+d zR&@sJ6~S6{7XDptSSPK=^z9@If0_n{z|o%;Cymgn+QLHjNa^6Qb9>(KD*3S>f`-AA z_LJhDYnYXAtNQaK=6)aP_KT5;_mQe!%JFaH^Is~`j(YdB$zhGUh5eH%s4YxE>9UuU z{yG%DKsNq5kD28qzx_Iw$@G%4n%@!Kw@v5WYB1*=K1g_)@_=0!%js9E0mC{f);;E0>Vgqg^YCe39A>W3*@fnhofY^b2j*1 zPCCZ0PW;CLPX^;Za^HeK4mGK5Z&=X|eWD7amF5Z?kKH6O8H5 zhmI+#=y!S+lXZ7ehCCj?oFuC5dPW?fzRnE|RL6naB~X>P3{aG2z;Ok~S8O;+m$(w? zi0w|QS>aQ!=ir?K?bq=TQW~7S)@r%2$+Ca3aEQZMr{QL=hr_xa7#V4_ajdf*DTI%F zAhLrPuKbj!W0Iu!Y{wddD?1oZE2qczDKUGBes+sOb9uX%7cSe~qC~>4wd~!9s!Z6j zcf{>xKvR`?1HcTP-G)<)P_ z_f!YFgX*y=FuPDX68n}ib?hst)Rhp{!xy$FpPvzI&EUR-Anl}+Sus1m+#Kyw*PanC zso>fk*~RQn4DKtKWSkx^%m(2%3(PqI5Hl_7<@w z+Q-zw0Otagx@&xl1_l{U%b&C8_;$}B|8{XxGRm4@gZud8a>r%Sw&}8XcI;*GT($ge zgb`^=+pny|Ri&+G(P?;^bow?sW>3W)67NnkTf9Bm04wT?*jG;9exb_XK7`;33ZE87 zbf=W3W6{lgTRfzK>FEK(!I%)3EYS*p$gPS)PEJ|i*tcv-V@x$z4oXmTmAx5dn=Iu> z`79HL(c~w0@14t|lrbWg`){Wv;t8YDeM|NHEhW;W@n^(aunxb$WmWHiNN5ezA6BN) zvzu-TU|eeDYRNTsFEjhf6?i}N&25xofQ=O-v2KQ;%%CdP+qQ$r6P`h<4GUJJWM2%r z9+1AhH>hOo9^Oyy3xKMVX5ik);+_kFxIEoeGk>;xbjQ=|BN1X&coitr zRK8(BMheA@p6ia)FGISqHhDF5QQrm477MZDDRCg)O&Y}%a4%jK`$e0MvCgqzzew(1 zc$G$?J2J6`4DUz^|4+~%Nn3oAEa-Rwbc`DvIm%a#Bk(=!Bt5-(xQdMNjsZ9B7H>jK z{Qx-fm?>g9CwGhmT}vj6A;-N1nQ(p8rm5Hf*I1t)Xl&Cgm)Yr&B@uBT2sgX{PRQh9uaF1QinoN4IHmEafXLTH^EE@-iIw3Iul-S?OCJ{&P_B>y4sk z6v3VG1AT|2o2732c(657pt|~IIGtQiW zxOm53+0=^;*$iX{i%&tRAq*UysY-*Z9=JZJTB5DuQ=G=oP^1Ez`e|fQ_PS&MK`9fq z_@fsoBwOjP-nFw-0@HB1O?C} z{h(=RQ+^HBeQe*>R8v#vx`J>v^@Yl8Og3vrcP(XM43o01j5EW7#b13$H^-*0ylc|( zoWLUeTzVDN373E_&xhcS!@`IDmu@oInKqm~!1nXee{^~`yK2BchEgY5A~b>mu7)4o zUtLq|vQhgS<2LGNt2G+n#F7g?`yO2Q&zZUNHUG!VTP>~l&u*iIMK^0e*I<%y3KpK~98O*jq%eCr$Y%ll$W(ny<9gUbn&V8C(uT9*nl;B| z$9QeHuc)c3#Qo*ADthRkC9*!?l)zd@@88~FxUzVxC=s=u_8(|#yA5M!;yxRg10 z%z6QsFwLTTxLH%#atCRu#$c{$44uTYa2;GH9#%#2OlEM;SPL0k8@UR0cpe`Xyad~9 z{ffb6c*|)mli|L&-nv|dGw^#>s|-?bj+f)uVi1Bz07Wr}1^+%0R{Dj^cE%UL4!26~B}Rb9P+Qnq}l+&LX>`v;&jq@m6^sMFyovdY46alzZ2pTXkj z=*@8Q9nT&h1`-|-b-^o2N@67Jpm*QGvmI(mqAj!}hLXf;4#U%@x;@teU)*H(()I9o z*7{D}lnMk3m)`M|AN}Vg7`snP?Pc+UU_FoL`@r6t@JzQr$IpVPPPJx)<0sAPT|D#B zCVnjsdqk~n!>NAHZ+;aAPamm{s|0!Tz$x(+s>Wnc<9!AwEOSk8f&J)d@s;e;;*P}E zO+T7Wi*G@^KJmB8V@=wcLV?Bd%mMU5X!6569L803Vu?l#mojzYNrZG$ZQvFNuc$$s zB6&U|#ECWzoK@fqmmv%{CZHY&P4vB&@+gSLCU^s9dVqExW}90B6oSXr8~yi&M)BKW zyMtS`cmqJ{2e~Y6%DybtMO+r29}(?StUwvoCnIoj|CfBUJ+@i$I2d?Iv9ExK2wu;3 z@aO?rP6l68xEFj@Z``zg7=gdWeXY+$;;YR0L)OY}_>TegL*bvqcKir;g{0EyYkvn!J9X!S$YX zOLsgK^VXlbpAZyN6aTmKY;-^B|KZs~y!GWnX1rZh5@r%WibNMIReZX}Gg!{hH%J0xs! z5X%78?|BP06O~I&&&;FNL%nf2?ry!_8(*~YvAC=+Z0q9Mh8=u2Ud|6|Tmj2a#7mE~ zvGDK1)ApFjhZAHN&!m8##S-vr4_NO@!ujU;fs7Rl9{nvjPr4lo0`^6+LkeiwZcwVZGGw zj2{k5TEAg5#iIH@^=2I;HKFl^9Nl^|VtsMNqIi19T^;3-Drx z9&JH)<%(aek7#i|=#T?IRRGIRX!m;Vt(+2#}A z2h|P5!#=h%ubcqNu=aVQBQK!L7x^&THK09cgvkipYgb?k*2Cl4b^Gy7N=c1?K{tBq z14?n6Pw6X(MAR&8zX-k-j&5Z-C|zN;w+^d8$x_2!Q#sV2AE{xl$sB6XiPW$k>BPQ_ z#fj!Hwi#(Hq9Nw%#{9}p?iesGE)+Q6%mFo1`?CFz>jw)?zJ5z4$Qoj0GK8{A>`0k# zX)H+H4$q;E0S<|TpId57Dq}uqR`9-Y%}EDcaD#Bd0$U4Vc*q@e(I-Fbr~8DntR*Ef|Tg+%!1b_Jsm4*rt_!Zux5hV)Stfm%5TJodaAYd0Djeg zTqy%7Di>p^-T-u$_RDmKin%vX7^c85VP#Uy|E?!^-tyn;-R1j6ERXy~oGt6KkGCeq zbS(7lT3C}|Dis*DB2Dr>_6v%|;nl_(ctMdY6HGp9XFS#q z!0_SJqMCFlt58#y#{ho({}7;x{}Q0a8ksQEXMMOgPR>_=q=yHSAnEDD!U4Xg8#xmk z{>dpk#UR1dp#Y(YFrv>xaUn2kaD?lu2NLl8=77iJy|1yNCdH%!dyJkERR-)}8mg$m zT)M8I51bVdypm^hpkU{!Y$!LPSq>Zv9|}9!(LC8lm-~oks_A?K?l(= zXjG=BQ-CI1?UQMGnx5IJWenv-i8A|Fz!5l^se|$nP)3zPrx^i3g-Jvk%c?-FpM(Kb?gAZ`04Zw~FXBmTcy<9#m5!(kl;7x>U zQx>9b^{jZur+0vwxuvboC?4>>X?CZyn`L2b&nP}NJyu=ZswkF2lXBac!~YSHzNe#~ zF~%V?Mmz2eF5(}9>kWfzf{&suUd02p!50}5-RF#8v*uDMTqb5X=rALo&1e`V8=
Xf76%AW?jX*;xy`kq@3)99rC%UnMbkRn*n>HS@8c zimc_aPIE>7U(EBGk4pDm!4oJReo<7sy?anx(X$CoCHQ6FZ2Ipy8+;%<hX zZwI~R>|r{}$l!?SFN%t2rs5uPDhDTMp^XG8^v$q~sKl2n;O>{HtPZo?HhpgX#B{;@ zY?16+1Q^kd97?{>6lq*+9Bs@sc6g3-(V=Oguiugjy_YSM&7J|wjvp^aS)F#Nq|tE& zO42%tG}s>8S*SD1(Zhr|jnVpaKfE*p+Du{E3ZrQml(usL(#AOy+A**%q_z~$masvg>=EXI0Gj~SEBSH!2;F{937yK2fWr-GH@K*?pYnM9?HPn)OOD|0XaMiWWh~^ z9131lQ=Ihu0o*VATbZt#ou%BJqiSaVVrTY}zS02uN}-5>ywl`(Tc zvxE?5?WesGE#-&!i7WW>5f*UPW4p^}67VPIU)l0d1(C)}~kgwLUDcg|v!^dTi2y?*$uI{V1aza_YUA z#%hEE+f-ABixQtn$6@h&Xh5eqN{B&Lh$t^T@POA#IZMB6S z7}IUkj-w1vC*mNTGC-1>tl#G0K4y(=FleYayi|~@%Rx)c3Tdf&6V~4GMENWnXT(Ew zL)g2P)kjAEY?nyQ0NYm7(#8`)oc12&;VP6Sr*D$D^(Ol0gkGHv53)^z2|;VOy!S}Y z=_$*Yj98Y2(yf*Oxc5xj-G99IR_v1AqQ%BJt!K*&&If|Chz+i77Fe%`%1|1U&W1XY zZfK=$D5rrOqq-o4N`MGD9Uu85`WVHvd+|C$wWP4 zN(O76G1|d(5LQX=55ho8==A)@F9X+s-_y@Kr*%`6=`!7jln^+m8=xo-Mk#GnIp{*u zMOAaS*_o0xGxoj`l|q^KTf&qVbL&W)}BiW))ZX3ZId6wY#V$n*pJT5 zS~>_=s&{Zgn5xI%Pr&@WVh_SMd__jO!lmt&DRhl20z{SDKP;g9WK8KG0;eSOOsTx!D(Wy!~f2onHk?X*Xri052U5tGpxQv-(YP8Ss z(rx1RgnNzu!xK;>7??2YLql;CbM=t*k)e1WbM2V5cRse@SFOA9;p5rL6oHY&%(Z@z zk4qR|r&Uvc-@wt3ex>%1O5lsR2dI0`Cfl|x&qdxi*XkFr{`?I`d%^at!x05s?+nF z^?Wgg|JPw{E5@hs+=GoJxB_QC1uv_S1??zy?XM|v#mt0@5_n5g8+KZZu=IBPOJpnz zUOM`nR~Wl1nLBfvqd8{nn|911B`8mL#4!wp%tdg_Tr!kA{O ze*&ITc(0xKfI}|T?h&XD#i9Zn#ArKV0S7C^WIYPAb4iK%n> zIe_JGfskQ+Vluv{d>ve>KF>+(^eJ#rHS=K1RIs|DUJH!l;f-h=%j@91?c>&%=~x?m z86E&wki&>Lu7czL@F45x>39+DJox@}9Er`sN3{*7D_I`COqblfHHaBdzWmN>$-Ns7 zc-OifRI`q?@fXAf?vpzx<7DHU!}<~%{DXyPFmC1cGV#Nt!gk9FZ0zHxSjsJ-zOV|o zDfW5}cT!2;>FFgdYLs$V$obu40v~Nqe>Ze`7926f)KEw8hoDeU&oc&BuyEXZ@I zzi-Wb5N~JZdP(2WSytgeTm@WFQI2bwiya4lF2_BX1cahKl#kbZB-_j04}w7WPShNG zNbKQ5DAO7}3pbhn@Qevq_|4!I8qRC@c$obtc+?ri&)1;rRy*tud=)%eR3a(9`@@qL zD1tNm4^K`&?)by=N+`7>z`}I|O4g14(-SYsu7E>Tk9j5B%BPK|UavnqkNE%avse%UVl!Q$EYxQv<7d5|&U-3;@a_uw0I@g$sm{WwA+4l#@dz9_;D zm*<mF0eYkVxsQf>z3J$G&Vw*ZUW+(@_j)UXI1Lm(_8uN;RYRNxim$qdhu-QT cP6Ng7knr)chiwNA{+7VLAUmk$SWnXb1C}Q?x&QzG delta 20738 zcma)k33yY*_V~=r&DugzTDq|Hrfa%DQ`Qy*ZWk`af-QSdlokZaCP7eJQIkqRRFvWX z0dYY^1@!^7Pj+lkP*CtCQhX>VL487tK5s&4Zk8thGfD8h_j|wZ`~UiVlQZYc%$b=p z+c{_E)?$gVSYl!x9_{>0hUlLh)8BKOV`3H6eGt0RXLEKxg#%*I8Jq9vcZas5pFBDP zp#htp>o4aXf+Cyu-ie%@WWF0&5Gta4kP<@4ishz9S1;feuU_y-&FU48aH}3&Wm@p? zL)@y>HH#K-tClV{am#B~Ev5M}lx6>|tnf(9qU8%7Uj2V5Ov|dEY$0VtE1s}m#qtHK zAF2O873NS|J4wRHIGPmV{)IYV6bB&@!T<;y1p2q|@kduL1Sm5i3-migPlGfAgj@(l z@*+--84j@m2zNu^AzUISaKe~Uh?PN@1fdebbO^Mu^AZ$$F#TheLCgeUHH10{4ziNzo4oj8T0%A)wb2$m33LmDMhF`q`~rDWg^dtv zg0Ka`b_h566d{^>k?GO7R7i{GuR=;4vTW&9gxVo(*@{I-5sfV8U|I}=eDXM8%98Jy zL2?6&&fExTNz+M#HUTTi zUv*!$JQ)**$$=PEa{uWT^x7B;a@SZ;%{&Vf`q$rzc@}HKk9e?b!IPm#zH(<)19Gkm zAxRV0^=m+H^lw08OqNg?pALi*8xE7>E$D?+7PR?c3rdATdkqcfshoyCc~1lS(s~Oz zvBrYtx75du!uUnf92c8;dYc8U*rq3mr4DHKo9O~ZF#n!}-{=tyh%RFUW1kxv z#_1A9Fs!<zVEaJlX(Et`f8e_N&DGw!}Ujh2|S%#FJoSzlDlWF5l6NhrpwxnPG+WR< zfE1rPKFsJ`1pXRJ~<4l?hV5Y0BOU{MDQUMVfw=n*c5@C5jdbSoIVpEEw=?=(MJ~a z6F@4N>M0gvrWO4Lk#Xd1jT@e^J zD=evzRbl=IW{3G!-4Emc_6HU;PZLh~1R$-rcup98|3H{70u0d~4C5~Yq&+$qf#(3y zbj92-JrN+q3nI`Ip??^`e~sV|*O)En_rE~T=Z6`0F9^dv3&RXnM_@ZZDzcvJO; zo(7Os)Ng4RzN-ttVP73ea^vs3XiU za5GF_-#?6R0Z1!w+_IoAj=_G=6@~@fVK~PVh6ZmK4(JIt{0X)J@|!Jao1!7qll7_w zlztS(QXPhqBJk-5Y>Eog0}(hgI*hN6z^@}vjKF@q!s$;$;42Y$DgwXm4e6ty!rpxw zP|ZmT(quG*#+Hx3(QwL8(U$?FJsh6f5ZWP64GZO?2U-%jlC}&BEfdohG59!X&U{X3 z^H&MIM3xj8`ZAyTNtNNA^q(Z1L$S9~Kq+!@onPaq6ncv^{657D!2%277*>X7?-UGsCbI8%27b1Ly}BFoFeQ^>Hq!MGQx$-4_LC9eU+Mb78t z;BCZ{w-UcbmKtv>*ZMg@F51YV{DHV1S(kqSpKh5wv>eC2;#uIZb(~3Uk_&1pbsf&q*=1r~kJfR%17Q%ec$cTVTYMZ+GgS_Q5G{V< zWgP+csz4>gt)zcZa#WsV5T*#lf=;}Pj4wLde`gOT-~%Uah$YdYqZQ)%Fvg0Xv}6^_ z7>vl^(w)plKJsPhz06L3OTY2GRr0i6psckFo~&f>%Vb1FJswB)SIosVL|gfsvQLlm zSA|$Y-l%*HA0T6n!Mxf<+Ka4Qj9XL^(D|B37Iat&P)#}7?NZy3R z_y$<&a=zO@F3iX?pX7^#icVEpxJsXA-t6>JzPnAgv^Gp}dET$RS<81nqYLAmo~$Lj zOHm!he)jOL1=X`7Q|O9kFQiSZ45wZ6bO83ODU5yX$y&;HDVOPBNAr4f9O=l$h30iPeicX_fOUivWKeI$Nlh**g3vg8=!z#dc15P5$ zmIA&9a6=@0KHwh$-V;f`AMgW!4~p=c2KZs_qIBHIcaP)3bxZ`QZ34LiW`^lj?1>D~ z%?+rn_6I5P!1|94Hly@8LakC`=etK3(cnTMTTr!5j0if*Lx;N(hC9uJN9-%^wA0zM zy@w7g47Ob5;9d3M3V?qf59KM}u5w~k)NRXB%Xo8_cOTzflUX6gq9|yEGDk!_V?eI2 zEWGOmiK$8+7|mCRNKM;YDdkl1F2qOT)x4{VOs?v0eh(_RC(@v^QW+c%dnRCkclC=% zqE+HuUsq3wPwIA{?v9k-O=U|Z z_Yx4f2(c|9@J68R3e&pN5N)(hd;(~{MDlfpIs$7IXtoILyGYrEKx>cCNTiW-fOaK9 zTNCNpRG^t15#s1b;W8lJ2(`OFh=QTvC0NX^2mVb;qa?Kxt0#f@XYY@4;;3UznbzEu zy`S$+jx_3BKS(-`?{0uKi&)zD`L!vqa{Ed1m!-gh_=F30sE^ccme~Au3F-N66d|gl zcBvF-JkZ|K(TrL#5ahc9#&DrJe~F7@dDl|Y9p?I4t@wNJ&05~IH`H6L_-pVQU@g_* z(pQ5ui+R`dPzSZ*rQkKdawBPGdvN}elqG!kz)Vm4SA^9`z-tqDS6QUM zk>H!lU|L2}KZ1rO@ZpB#`Z0&odW#u;IB_92!=XBCP}<8~Rlu9!8ck|(zqhZ0d8HVX zlv0-twYd=a`MxB1Mvq*4hT!{uz;Wd2{r4+-i|++_S0X8za~Ix6o|7a8?n9&_7AY9D-9l?i9IMm*>xKR#H-Ea)MI zxdK*@6?2E-3FPg$KjSaSv(@tx1_lcrRyAwHiyo`A7h!|SUI3*YC%7g-f4m1f)=PL- zf$&AgsrNNvlczvn^v_C1_B<;+y-N+%71oSl4hBeF4Ug|8M`|8me)o~Yc^n={3g@jg zuRVa9%3UIeoJygxx!fg%eNnFIbYB*W2&-**%GQgJ^RICkG;lmTZQCTA@>}cxC}q#1ypR6ejES)~MUP zY9`heEfhMRd8AlGlUT?5{C*FPVNrW z;IvnvQPnVk8GY}%>Sk0YxS}!Dy2}=^UWItCx)v93omid#0&ZkbdYdw_6%EnZm>n>x z2&KpP(5Hx-Dsej2i*8{fmD9kI^~JaCbbH_PQFM3WfWuCeR$5^0^6w+<58aSqPDV~F zSjxQZA=wLanT;M&wXl(y=pjEY9LnT*NS{U1F-sOL%42?$$o56^Febi5&tW@RSNj5f zfy69cgLje_7w6*IZ(8rUO zcnw*!{C@l$`D{7C?~_>%kHag7_2HbP+1~Y%kxu+VG#JDI8zjmaVB=jJx%}|J1i>=` zlz#3|%G)4=F9 zrOaj!@`Wg|rH9x|2Ib3M1GIfB37Rtd>w31r5b`2M>i`-ApL;?8rE#F^k0(Vd2Q$B& zAf}axn2_gI&cgN%kY0b}OJ@T&1j& zj3y*bBC{VEhVLgYK9U4B&U=r{($DlZ2nEm;t7oim#EG;sp^Fbe7md|OJY}KXPhK@x z8z;d|vlpejAR!)!eaZc+lJNs%&8i}Ng`8NGoIKA{P{+gm2}0BBd~%)7{hNQ(wE_Xw zMG+R?A-+|a<$Hs)v{lMS$iR#J8ve5c5?}LaukD200;hra)de7~6alrO*7~}FI-KHn zKNlc=seD74}E>wx}euJPAr;+5RCg6o+(NpPcwCD_yrl9D@wWpS<$9otDqgRSc$c(3F<65%+>Gb}iQjosQWntPnWO`bij`z8ZhdE%p zl>G7Zv;AQsf+1{_-iL{M59lASqEpf?WE8w^u4S{?jBz!IX~@ILq^KdO?_B{-V5tRM z)ZpMC%P=^rIkKW5AAd^TY^VXr%buB-FvY`#<|$a=Q^RwH(Rtm6$oyyS!t==+&-BHU z$j8qV7L-BNjgqli304^=R7z#Z4aEb%u2zW$1OLr|O3Le=bpo}Lw6(kO_vGx_a@Ajh zl)>xQNmFAY-c3d{ju~(ktaPP##dC)vYYR(RCEhwgUTW-zmyo@UC3qwGz3~M+nbfbF z2nxby>qg+`iD%t9yoZ?AX926%o*P4cUH=f)lk#U5;ohX>+27SkeuY4pzex6O_+2^9 z_m5`YwYvp!V*CWT{QNLHifCUbiXI1bhsSmOv6ma;fc?rBl3o|H6=2i4`drvCl=`;~$bd_$vQTU_L?OVa4o64GI# zT)c<0Ha)87=<&L{0%Y{&Y#c)tY<>gJZjryNWk7O$x1{1{v!90pRdEeo$B%eN&ZjrQzO$}Vznl7Mf^(^rhfbh0bNA>@s11Cvs`|4JR; z{a0!{xwx$Z7GbM(27_8#;{8Wqcl|l?|z?@eb@I>;LR||7X z{V2Tx=E;{GXhuOD5`Cz^!~yI-Ecw*V{7ujhVR5O0PmtIwcgNKiz>!qr|QLmRENTXYh+H_P1tZ{4vSd z{VdkDwCvuA@z+-R^5G-@SJMGZ(?> zxOV{V+w#@kr5LxhjQXILe8hDBLCJx1T@B_qn@;3VjNJ&9b82X|vSQisZIH?<6z%__ zO74u7`U6@_9dFM!4^7zao8~Z%RKfHI)6w?w?%IV=Q%0KtIh!pw%~*#EbYf6zG^tvH z?vx;G*ktD+y?IlEjFXA$d{?g<>x|9+o`(tE3pUn-?w*xjLo3=OQw(S))eY5%L*jl< zxobll@K0;+NW-dr#fJg(dRiu4ym zRbQ%2J`eFh=`k=e z{sK<%BcKw^__a-$7n-uAE)@ec@eO4EF?IZeeRSP|8J`8MP;K?TBSI(dBVQkjHxAvW z^l2le$%bcvLA>oBbkvg*-dRkh!pDUp|t=VH>*dVD^{e<7)#)a18w;PnP}MKBwRu)3e* z-UW+uSH#H1h8TO1s~twU2;8KG<|5Z6^6@8Gu%NDflFsb&5k+eTOt`_VJHerTw)Hgr zoNQ{Fi-(av+LD>WbBONbP)II0c};PzS8@*lo&M9hg5NCIfW+_N9*`GDt>sxa0bpo7d3@0-& z;J$^qH(z4_Jk`*u;Sx=s-fGmbsjR`!nkYOkVf5*(Q`{Yl0gUA*cb}Wb(h?~JtukS9 zx4)Xpd!DTPY>An_EQzSHlT}39s3HijR-7D4Z2haZQEYSaEaSjw$br&i&T)vz;-GTY zhs5!c9(`{IFU>+YA-HqWIp`IW0Xq!za%2M?j0(|0#j00Tsj7YMN_<*XIf4J7GxLl!IE2Ni4#~58#*PR2o=d-=@kkjO*IE-?5}k zQZ*fQVhvqjw$K7omNMJ1QDrc9S@u(VLn-zrD?U#(f8oR18E9x(gN$m?6+!pl!#;P4 zj}-?7z+-v6b>fk6X8{XaiEmXg>ynbiQ%Z_^pdofyCR`z?j`91?&k~nb(ot-oS#o)jCG{! zi(Ji7utA68^6o|xv3=1m*5uuDdd%YSQ7It4SsU{mD<;VVhhQi0MDrjwSnhXu)q$tOaw zO&<3tB0D<;G~F3zSK-IV*|Ue3AA86%UseyQHI6BzjRe;P&k1ffA5;hixQ7`EP2CuL z`K#B#o%hC9U&VhL;JXJ*Ia;br(Kr=!)nk=-E}w5U>mhYt&t!IkKjQ0$)Nxb7yq1$Oe;boJ5awM!IcJ;Btnrf+ z`};VZoVOQ(Ge-H%1?F`hx$@1(zR&n*h2f@#u5qASbtLmb37$!oUig4PaPw#{V1D+J zsqNq5$4S=3QGM_C(mD@7_f*b1Wu}lvFHXRN$%%_4IEvtJKVUxgkbU14WNi0nZYmL{ zH9)Aefu*nz-(3A4PK3PPCz^Ngb-@_CXGR_H2{ z|3e@?P6kjF2YbBtUkNc@6W+M#UfGJWkt+NO##YWQn(gzEcc@acw*p znox@Q%}4m3s{0&+5sUt}Vl6(RyEc-s`bg(b)A2Gg?&rnHWk%IaP2`#<1N>#(?kirU zPzH{j(anG|y2AEUsEjrw&y)9xXrU*|Dzc}V-OwHRjd^;@`qOa2XT6tt1%jxqR6@<&Gw zxE}j=7RAN}zlE!~7*VBF+gT3j{qFZ#R&G2 zsK*D%T5$}xOHYaeH7`ljUvk?Wf1KP9=iqn9cxQU-1rKd+lwB_2dV0Xn4d*H6n5em4 zx+!5!A>Q3`+NsC-H9dF?<$u$z3-x`z=ie21zxxPL-b&4`4T5(!bQs)rKX!C*hgh-h z09Bqq*Mr+Vl?wE=D3n2`DyhDeW~O^3s9-SF5U22Abs4NHT5pmB>rKf!e`?TkWhrVW z2PXv5C4rUk9Gv*OV4fqL`X<6^=PslWbA53C8KTXG9A_m1lN`dr`I8Y`65UCyr24>F zDR(M2r%j`bN!a##|5 zQI8q~vp&R-?`}u5M~14?QoD-pw*86O0HZohIPDx@Xa2;_0Cwt6>=a;ae`0Nb{qiUF z3t)~vF$Z8*!k8HUrp`^%>`$JT-F&w_J+?Z>b;WvOpf z-Cv8Lx`LEG{ZOjhsfz~osT*%75sVUCh9}0HmQ;nOrRc)b9Vx@0PaO0U!K6e8YC~qI z$^fp(UecZ;>=tDjJ!$D=fXRYj8tBMTM55=f4vltY+L?7xZc%aMu*8q5e4}hqlgWB4u1rI>0 z^fIs(nM9uCyZf5AOM^g>+W^1YrGcT?KyVyOZh0V{`ktbw8b{s53MW>WyFT2~4D6Ax&lHQ}h)>GGPmy#_a?~Fo_a5cgLMl9Kz`OXBXRvb1 z2@r<3nFKxg*h2EX6TpR3@9iIZU?`kQ%v33ZlRL^l8)a-S!#|^cKpy9d%fEglN5kCPncs3(Tq_ z*)%DIXL6Ca-h$!xS<4)FS%J7DhZa=j&}K-T%v^$xff8XHr9~)ku%*d)*8yVo_wQQ; zR~_Ik3B))ygO`rURa~Me`~3+C^E~me;H!q5HJ+hHj$tpxCrkN+b53 zRC7qc&NM~3qY9=TlY+%#egt3(j{l}kQh>df@p0|C zWUOSa;eS=o zBw3<$BAg*z&+n&XAM#B2Pq`2r>^m$ZQCC+b$boB;cv+j$+?BikTiB6Q)yCSQYPgqPn0h@6 z)+Iq1IWZ+Dx>z2hl9p3DaP35Mk$TconW$+V3sAe>Q2Z~!b|(pV|0L8eTah` zNOuCJd=Z`QEIhb@?Fwe%<=|a-@74qiuL?u)5nCU{BLuWj*R^)894;}kLOxKRd;O6; zgIMP-1vUD4HQ&i21?=#)4h1;tG`922%wU}j57xjP76QL!&^^zuaBLNEzl3o?UatmQ z_Zrx=?4QZyRz1eEnA8*2CosN8(WYctbsMc;Vmt^Ru!1+Lq1RMUneqiq=-#K+`Z$Bl z%yqw2VnR6*WH=Kaw~muxH=bY}#bOfcgGminPN=5(R0rH$3%A^}tv|5%1b+O`PB~tI z6E?vHfW)ajJeUO!I1+1oCvF_z>#TYuo{de`HA+nILsqQ{+hgtuEOta?FS4`UXMmH( z{ejh~!h>U92Tg)L@M1W0Qyx#NsI=y*aesWTb+Q^KOh^o30UN-b$Du4$t4iP;cNm

d<1OWDMl1)=Ukap$Dr{`tmxvmkQtQ{@HixdN@NUbHy!S`I-hNB`uQng)&L* zG1A`a@Fca+AlwhVc*FWP4Sv$BWXNV=gP;&#_A8Y)aZbSfRY1kTGe14d77gqp)4zh7 zx;7WeHrCZwU6xo5?mpf*Exnq1TG~DUUUcZ-H3xTE+70-w0e>v9wiMRmtUP#&7J3-F zs|j`tf6DDnw-^PKr2)wv5gyG(x z5QeNUh;nH&6t*YoX(2`|ktn_u7CFk~JJp~JTB7soE=n76FG@|(7p0AN$N1PctwW-5 zO8n10W;@r+3t`1~wj*r1Tc3``-{Lja8NKiiO!g7$@ZR{JLl2CCClv6&ZI+Y{&X*^3 zs$yAaEd<x&9~gcBo@ZC+dVOOXu1|JWbk(Du^3#!WCXYM!PDgjhVicB z*4cgVH2k*pFMV)1{?Y2`gCE78SXad2{>+qsbz>~%@VnN(#NrR|V~1wP;X{nII1yKl zK&|2PDOz|&V6tFg=W1-bk7!_%(>T+VDu<@KK&Nr8R>~Y2*gG`NNlGw1tal~j0cJJZ ziu4wVkNvv2aO0=9c%&-f;ZmcZacKRtL9!g=`ow}$uHKM|%**i%$}P65l>OV`b+X>$ z4)z!w1+|BZk@%sd9$YE~u>X9~fga!Opob1GzG+c7RfrMeKzqoC+4j1_RA1t14?RUe zjHW6Dwn;%#SqbyFHc$}RDhnF_-oaD`+SBa zdhc*L6a2eW)$8xloPpR_1ls5qfPDd&v7RZQyIHZizsP_K6M<8Uw!5c?{x0&+4(rNo z<*vh_C2s@mFx+?0L_&SXsYuj&=memWK%!RKF1;U8@8CDIOc9q4KKxL;B6!_epvQOj zV#H2qVSQ$8g~+It)&@O}1y>AQPz0>6fL=P35nH82^%;jk$+Fk7+W^fif zx==E-&55l=>S1-&^)m59!0Ly^lcEG&2QK)g|G0 zYgq!$z*DU&67U7<%oKc20TQF6vCTF0skH`BRTyZui7$6B2Tb&^C}}vbUFa>yh0)C` zVZ{{-p}v$Lano_@fkd3m9PhDSNyL-Q6>AUd5aNVhf>K}%Fj@29?lmvERSnY>?ok|S zn5@{LhKU+J;F91VsFYN2p#%3J;;CSX>u!Ydm@Ra%Bj`Z{e@uuKSVx91U%-wB z1(`6Sx_>RokzK|kUNjfqyzsh%$pg1&UbOAd5$N~IXxgOE!obc-4M-qH=1@W3KnhdP zX1rsHq|r?tAhVBh!0wlhHp9(P7Cpk4tR%!rh?C_Jf3=`MrQ~Uw9G5%6dB}1*8&Eg<=Ob33A#Und-KsipmMUQViR@JrO%O zW|gpc+6`0k+YnxkK)&F%{dG`fP#!O0{1R+MIy>G;*jV0gs+AXI zp*+kEc-Rve)qDS8%}Bz=J~RAA80*mjlL{_ROVGV6V69EU1!ki^b0wXuVguFkQ3?Yy zpWYys1j5wt=9w%s!FdK2?q-#Z8Io+nLsINY=Q395oD{`5;0r)lE8j|Y3vp7Zb~l)g zI`~*Y4bL)RL`2SiNGgThg+cEDq*rVS4yAJolwBRgm>u-KjZ51J@475Xbvu32ghi$Q z7~r(X2LK_5hTA}|5;)kRILmfPUTFk{YII~tn+7pM@?lBiKbQ1fb&v|h4Dyl8&vaBqsrd?CsL>%BEkBh3qyDA-8>%9JWIFH13xF4Pk9pJrQ(1BQpD^ey% zaP!F_@pBL6kp4bl1K2QVyG(oou6brdCqg)+AM{wi?T<&B?>sHc)`14Dho^jvWNEXGq?zfRABJldpl zI@Uqw(o`74ZplzP1!VA=6z2cD^{at+P<+#o@pZQEq-ba@qivFs+oxw= zo5wmc2Xp;yyN7w@2gDOBn5D>}5LETBY-P%vD4Vq@2gkt!TGnpd$ah()jg3EFhiASr zH27ohIuQh>jBSLWo(Dr~JqxVN63y2H`DE6&u4DbU>x_QP_3-}lgJfvhDI~zTXaUPW zcd6>%PCdzU7zQTZMn#$gzet;8@8h6Q8=}!)2By`&m-*47cCOYa9cpKq22NGyBn@`N z(0!)F#j=zUtVOoJ^|^GMWuEStDU6gjEl2xciFM2rD8GMEkeJ>;izq^M-#M$pr!;D! zE?)4X!Y!(CLn)kNnCG(ww4$!YDb2SV=MJ#v!h2*?Sp*XRiLV4Nf=+euPYvrdU}a|E zG!W7S+Oi^QQm~SHe2;Mh>#$1i>u}+-^jD$0$iplSy7_={11DOL(E_hbVSaoGr^{FH zss&!k>#!5!X8~4pUtNhi5q=+DG53Y{5GjZ|g0_bhiUBq{_jR`KrD)w*^QX1nn5R7t z(-i?xvdiHu`j0DBywRd|D21ZhlM>qXC3sKTBo-htdE7F;af43OX~E)!t=ecQgS~aS zmyU%dGz#<=YjxI58F-*Mi(ylWD|bqA9$s6(;dGCX6H?J&@gOQ<_%aIKjNehyz%6Hl zyr4603s8nHNNgo9x-9fTNU4j12PX^%($T<+bq8HT<3Zz8fj)0x6zM3nLS&;A^a`3o z&KL#ahS#y?TEDf%K2%Nd6(9XZ(a6U7RrPQl!^JaRYO z@#hJ6*sAf@E@*BdJv1#!o@bXV{7K1j;DJjD2J5fhP>KUhCfVFM8Yhb%sY2O)dpPHO z$M=wvmQiHFTH?NhGO7nd>fF!2TK8n)M>2tFDs!`l&cY~^xnlv+Cpr{*0hF6CGX*eH z+|*c$3^;yC9%B!w{I^@ntdEIg=}+da2c_c4Q$hM$G}-b-7o?7(c^FrZk#_}V>?y* zi}`z)oo^M(zUyB*f4pQzj#q{IYIW7c)m7DP5^7V0ys|;Jf?mt?uh?{;L7;nCv8!E! zGEk>FG+E9|aZT1egYkfIOOF+~*pT9T$q1$(7-#UA$95a&N_4Uzanahjon@xtcKEI4 zoo_dMFWKhyg$0Jh#s<2B#SFoETxuOQ1ZO^Q!@|QDYc{hxqMA;2Ab5|f*^4r|+y~%p zD}0RcYxrwNV2Ypbh+cTULjyqtL9uX@?Fg)o{_snU#6D|jeg*hL8SdFyBPcARg;eX2 zA-I3SU^o#?)fn7|;Y4aXjTUmOze8efTLXTC$Aopb5&Ts~>og-yRl!Q(K9krs>v|xXeQ3QT%Fl4lC6ooq zcm{kN0r!`%HoDzAdbZIteN5=(OPfo}Pz{jg7neK_-xnuhL@(#yy+RBOr3PNsBr>r4 zG0;+COlS1K5%;d){)*Qz-|ui zrhwb+r9I>+BZTM8?Nq_sdBQ@+_&ys|Ug+R8NfJr z;I8oY3g6ehBUP);4INp$oZ6UnTO5jdXN?X_|v@o{sfhuIFh6t!L}-n_;bLg!6B_#5d* zCwGhx4v+2wcS4{ZbBXYB5fo?LS@@_c;I2Dinqd0))DpRHs+p^24DmK>h_x>Sw<(j7 z4_#HkOKLiQ`t(%)hik4_)RQ9fhNqFOFH4jl%CU>J!$)0&Kx&tUC(e>&cC& zpa#b_TF(^VQYNFv+Pe_Hh3~dD7vf>yng2&29)(}G4jhfM@R>ssM&t7g^N)Zvehhw6 zYYrT}sbFNbJxPUN3fCQaa}0h8_d3$^Q-8P$sdXm7wcw$gad-lQ>#VDa@I|JV$9hjO zeh|3tEygXZ2;Nj{bqP*{*lNl8WC_lO*f`1hehKcUT z`u;+HW&$!;pAhiN>SjA0CCf8+J$|U~1e_pa(vMp!@5OP}I{Gv79%!h;`td#B%6-(T zF2_6N_w;nT-?i>7$767`^?Erx172j!oP@U)CmCTV!A?vHI+O!id+kDETyU4NG!)YY zIf(TJnkMK_gk$pHE=3hw+VtK~w&G80FIz zaIoRnvA`~t^63@f(_-B`8J84p4(yVZw(213#z1u>_H2NQ#GVN_aA~U+QlAR!!nC+H zQvA_Fz3#y>>#XBv;6*ra=r1#H zFKnGO3$Mm*>&aPoEAye33@OdFK2rtni66Ays=^I;AWexqv|%=880*ISF}NPB@7#}D z;{I@}C5!MJxIpg`bV3sBM_95hn}hpJz7;G6f6G7Je+1=@Kim&W6#K({pTvS+6Qvl+ z|8QTHWS77OHex{uFBsFysZR1ww=wvK`=F#t1FbVxb6EpU=YP6yAG$CHpOi6iJ%_ec z=Q_c04fh|B#!AWp(j>_lXc>~vH$oXh=YBoBBa(6a{~ z!Vs842Og4DH{;)}G4br8gwyE_Xhuc@ngHPf#0LVLpV1Kde=bo{GsDoZ^|SDQEVDY| zSrtyTHpjCGR{jdsS#=4lN_PHi=pTn2GA6JEU<^EH`-s*@{{a{cG*16)5T{`Z(}2!c zPbRSmF$g0h0f`1m7DMDX1Z6n*7h*I}aw0(bpUGlDX(@v^4V2CUv^M>K6FK@1$7l%G zgpdW|G)#tEbpS)9p|U*?r-9OU-JwG%8^md#^fPzpP@4pC8Yo=^Xf>T z%?NQC&Z7oo0C=aiDu~lS>6LfrP|5;v8YsQ)4jpRS192KCeHTDF2#JR*1K8(eR<4n0 LI;7@Uck=%ONev<^ diff --git a/drivers/gpu/arm/valhall/mali_kbase.h b/drivers/gpu/arm/valhall/mali_kbase.h index b260be8077a8..da34a4225975 100644 --- a/drivers/gpu/arm/valhall/mali_kbase.h +++ b/drivers/gpu/arm/valhall/mali_kbase.h @@ -45,20 +45,11 @@ #include "mali_kbase_mem_profile_debugfs.h" #include "mali_kbase_gpuprops.h" #include -#if !MALI_USE_CSF -#include "mali_kbase_debug_job_fault.h" -#include "mali_kbase_jd_debugfs.h" -#include "mali_kbase_jm.h" -#include "mali_kbase_js.h" -#else /* !MALI_USE_CSF */ #include "csf/mali_kbase_debug_csf_fault.h" -#endif /* MALI_USE_CSF */ #include "ipa/mali_kbase_ipa.h" -#if MALI_USE_CSF #include "csf/mali_kbase_csf.h" -#endif #include "mali_linux_trace.h" @@ -84,7 +75,6 @@ #define KBASE_DRV_NAME "mali" #define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline" -#if MALI_USE_CSF /* Physical memory group ID for CSF user I/O. */ #define KBASE_MEM_GROUP_CSF_IO BASE_MEM_GROUP_DEFAULT @@ -92,7 +82,6 @@ /* Physical memory group ID for CSF firmware. */ #define KBASE_MEM_GROUP_CSF_FW BASE_MEM_GROUP_DEFAULT -#endif /* Physical memory group ID for a special page which can alias several regions. */ @@ -118,13 +107,6 @@ struct kbase_device *kbase_device_alloc(void); */ int kbase_device_misc_init(struct kbase_device *kbdev); void kbase_device_misc_term(struct kbase_device *kbdev); - -#if !MALI_USE_CSF -void kbase_enable_quick_reset(struct kbase_device *kbdev); -void kbase_disable_quick_reset(struct kbase_device *kbdev); -bool kbase_is_quick_reset_enabled(struct kbase_device *kbdev); -#endif - void kbase_device_free(struct kbase_device *kbdev); int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature); @@ -242,162 +224,7 @@ void registers_unmap(struct kbase_device *kbdev); int kbase_device_coherency_init(struct kbase_device *kbdev); -#if !MALI_USE_CSF -/** - * kbase_jd_init() - Initialize kbase context for job dispatcher. - * @kctx: Pointer to the kbase context to be initialized. - * - * This function must be called only when a kbase context is instantiated. - * - * Return: 0 on success. - */ -int kbase_jd_init(struct kbase_context *kctx); -void kbase_jd_exit(struct kbase_context *kctx); - -/** - * kbase_jd_submit - Submit atoms to the job dispatcher - * - * @kctx: The kbase context to submit to - * @user_addr: The address in user space of the struct base_jd_atom array - * @nr_atoms: The number of atoms in the array - * @stride: sizeof(struct base_jd_atom) - * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) - * - * Return: 0 on success or error code - */ -int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_atoms, u32 stride, - bool uk6_atom); - -/** - * kbase_jd_done_worker - Handle a job completion - * @data: a &struct work_struct - * - * This function requeues the job from the runpool (if it was soft-stopped or - * removed from NEXT registers). - * - * Removes it from the system if it finished/failed/was cancelled. - * - * Resolves dependencies to add dependent jobs to the context, potentially - * starting them if necessary (which may add more references to the context) - * - * Releases the reference to the context from the no-longer-running job. - * - * Handles retrying submission outside of IRQ context if it failed from within - * IRQ context. - */ -void kbase_jd_done_worker(struct work_struct *data); - -void kbase_jd_done(struct kbase_jd_atom *katom, unsigned int slot_nr, ktime_t *end_timestamp, - kbasep_js_atom_done_code done_code); -void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom); -void kbase_jd_zap_context(struct kbase_context *kctx); - -/* - * kbase_jd_done_nolock - Perform the necessary handling of an atom that has completed - * the execution. - * - * @katom: Pointer to the atom that completed the execution - * @post_immediately: Flag indicating that completion event can be posted - * immediately for @katom and the other atoms depdendent - * on @katom which also completed execution. The flag is - * false only for the case where the function is called by - * kbase_jd_done_worker() on the completion of atom running - * on the GPU. - * - * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller - * is responsible for calling kbase_finish_soft_job *before* calling this function. - * - * The caller must hold the kbase_jd_context.lock. - */ -bool kbase_jd_done_nolock(struct kbase_jd_atom *katom, bool post_immediately); - -void kbase_jd_free_external_resources(struct kbase_jd_atom *katom); -void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom); - -/** - * kbase_job_done - Process completed jobs from job interrupt - * @kbdev: Pointer to the kbase device. - * @done: Bitmask of done or failed jobs, from JOB_IRQ_STAT register - * - * This function processes the completed, or failed, jobs from the GPU job - * slots, for the bits set in the @done bitmask. - * - * The hwaccess_lock must be held when calling this function. - */ -void kbase_job_done(struct kbase_device *kbdev, u32 done); - -/** - * kbase_job_slot_ctx_priority_check_locked(): - Check for lower priority atoms - * and soft stop them - * @kctx: Pointer to context to check. - * @katom: Pointer to priority atom. - * - * Atoms from @kctx on the same job slot as @katom, which have lower priority - * than @katom will be soft stopped and put back in the queue, so that atoms - * with higher priority can run. - * - * The hwaccess_lock must be held when calling this function. - */ -void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, - struct kbase_jd_atom *katom); - -/** - * kbase_job_slot_softstop - Soft-stop the specified job slot - * - * @kbdev: The kbase device - * @js: The job slot to soft-stop - * @target_katom: The job that should be soft-stopped (or NULL for any job) - * Context: - * The job slot lock must be held when calling this function. - * The job slot must not already be in the process of being soft-stopped. - * - * Where possible any job in the next register is evicted before the soft-stop. - */ -void kbase_job_slot_softstop(struct kbase_device *kbdev, unsigned int js, - struct kbase_jd_atom *target_katom); - -void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, unsigned int js, - struct kbase_jd_atom *target_katom, u32 sw_flags); - -/** - * kbase_job_check_enter_disjoint - potentiall enter disjoint mode - * @kbdev: kbase device - * @action: the event which has occurred - * @core_reqs: core requirements of the atom - * @target_katom: the atom which is being affected - * - * For a certain soft-stop action, work out whether to enter disjoint - * state. - * - * This does not register multiple disjoint events if the atom has already - * started a disjoint period - * - * @core_reqs can be supplied as 0 if the atom had not started on the hardware - * (and so a 'real' soft/hard-stop was not required, but it still interrupted - * flow, perhaps on another context) - * - * kbase_job_check_leave_disjoint() should be used to end the disjoint - * state when the soft/hard-stop action is complete - */ -void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, - base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom); - -/** - * kbase_job_check_leave_disjoint - potentially leave disjoint state - * @kbdev: kbase device - * @target_katom: atom which is finishing - * - * Work out whether to leave disjoint state when finishing an atom that was - * originated by kbase_job_check_enter_disjoint(). - */ -void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, struct kbase_jd_atom *target_katom); - -#endif /* !MALI_USE_CSF */ - void kbase_event_post(struct kbase_context *kctx, struct kbase_jd_atom *event); -#if !MALI_USE_CSF -int kbase_event_dequeue(struct kbase_context *kctx, struct base_jd_event_v2 *uevent); -#endif /* !MALI_USE_CSF */ int kbase_event_pending(struct kbase_context *kctx); int kbase_event_init(struct kbase_context *kctx); void kbase_event_close(struct kbase_context *kctx); @@ -449,22 +276,6 @@ static inline void kbase_free_user_buffer(struct kbase_debug_copy_buffer *buffer } } -#if !MALI_USE_CSF -int kbase_process_soft_job(struct kbase_jd_atom *katom); -int kbase_prepare_soft_job(struct kbase_jd_atom *katom); -void kbase_finish_soft_job(struct kbase_jd_atom *katom); -void kbase_cancel_soft_job(struct kbase_jd_atom *katom); -void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev); -void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom); -#if IS_ENABLED(CONFIG_SYNC_FILE) -void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom); -#endif -int kbase_soft_event_update(struct kbase_context *kctx, u64 event, unsigned char new_status); - -void kbasep_soft_job_timeout_worker(struct timer_list *timer); -void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt); -#endif /* !MALI_USE_CSF */ - void kbasep_as_do_poke(struct work_struct *work); /** @@ -549,7 +360,6 @@ void kbase_pm_metrics_start(struct kbase_device *kbdev); */ void kbase_pm_metrics_stop(struct kbase_device *kbdev); -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) /** * kbase_pm_handle_runtime_suspend - Handle the runtime suspend of GPU * @@ -587,43 +397,6 @@ int kbase_pm_handle_runtime_suspend(struct kbase_device *kbdev); * Return: 0 if the wake up was successful. */ int kbase_pm_force_mcu_wakeup_after_sleep(struct kbase_device *kbdev); -#endif - -#if !MALI_USE_CSF -/** - * kbase_jd_atom_id - Return the atom's ID, as was originally supplied by userspace in - * base_jd_atom::atom_number - * @kctx: KBase context pointer - * @katom: Atome for which to return ID - * - * Return: the atom's ID. - */ -static inline unsigned int kbase_jd_atom_id(struct kbase_context *kctx, - const struct kbase_jd_atom *katom) -{ - unsigned int result; - - KBASE_DEBUG_ASSERT(kctx); - KBASE_DEBUG_ASSERT(katom); - KBASE_DEBUG_ASSERT(katom->kctx == kctx); - - result = katom - &kctx->jctx.atoms[0]; - KBASE_DEBUG_ASSERT(result <= BASE_JD_ATOM_COUNT); - return result; -} - -/** - * kbase_jd_atom_from_id - Return the atom structure for the given atom ID - * @kctx: Context pointer - * @id: ID of atom to retrieve - * - * Return: Pointer to struct kbase_jd_atom associated with the supplied ID - */ -static inline struct kbase_jd_atom *kbase_jd_atom_from_id(struct kbase_context *kctx, int id) -{ - return &kctx->jctx.atoms[id]; -} -#endif /* !MALI_USE_CSF */ /** * kbase_disjoint_init - Initialize the disjoint state @@ -730,8 +503,6 @@ int kbase_device_pcm_dev_init(struct kbase_device *const kbdev); */ void kbase_device_pcm_dev_term(struct kbase_device *const kbdev); -#if MALI_USE_CSF - /** * kbasep_adjust_prioritized_process() - Adds or removes the specified PID from * the list of prioritized processes. @@ -744,8 +515,6 @@ void kbase_device_pcm_dev_term(struct kbase_device *const kbdev); */ bool kbasep_adjust_prioritized_process(struct kbase_device *kbdev, bool add, uint32_t tgid); -#endif /* MALI_USE_CSF */ - /** * KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD - If a job is soft stopped * and the number of contexts is >= this value it is reported as a disjoint event diff --git a/drivers/gpu/arm/valhall/mali_kbase_config.c b/drivers/gpu/arm/valhall/mali_kbase_config.c index 37dbca129f86..c90952dd9d44 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_config.c +++ b/drivers/gpu/arm/valhall/mali_kbase_config.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2015, 2017, 2020-2021 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -62,43 +62,3 @@ void kbasep_platform_device_late_term(struct kbase_device *kbdev) if (platform_funcs_p && platform_funcs_p->platform_late_term_func) platform_funcs_p->platform_late_term_func(kbdev); } - -#if !MALI_USE_CSF -int kbasep_platform_context_init(struct kbase_context *kctx) -{ - struct kbase_platform_funcs_conf *platform_funcs_p; - - platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; - if (platform_funcs_p && platform_funcs_p->platform_handler_context_init_func) - return platform_funcs_p->platform_handler_context_init_func(kctx); - - return 0; -} - -void kbasep_platform_context_term(struct kbase_context *kctx) -{ - struct kbase_platform_funcs_conf *platform_funcs_p; - - platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; - if (platform_funcs_p && platform_funcs_p->platform_handler_context_term_func) - platform_funcs_p->platform_handler_context_term_func(kctx); -} - -void kbasep_platform_event_atom_submit(struct kbase_jd_atom *katom) -{ - struct kbase_platform_funcs_conf *platform_funcs_p; - - platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; - if (platform_funcs_p && platform_funcs_p->platform_handler_atom_submit_func) - platform_funcs_p->platform_handler_atom_submit_func(katom); -} - -void kbasep_platform_event_atom_complete(struct kbase_jd_atom *katom) -{ - struct kbase_platform_funcs_conf *platform_funcs_p; - - platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; - if (platform_funcs_p && platform_funcs_p->platform_handler_atom_complete_func) - platform_funcs_p->platform_handler_atom_complete_func(katom); -} -#endif diff --git a/drivers/gpu/arm/valhall/mali_kbase_config.h b/drivers/gpu/arm/valhall/mali_kbase_config.h index 2f9e28aaec9a..aefa8e631650 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_config.h +++ b/drivers/gpu/arm/valhall/mali_kbase_config.h @@ -33,14 +33,6 @@ /* Forward declaration of struct kbase_device */ struct kbase_device; -#if !MALI_USE_CSF -/* Forward declaration of struct kbase_context */ -struct kbase_context; - -/* Forward declaration of struct kbase_atom */ -struct kbase_jd_atom; -#endif - /** * struct kbase_platform_funcs_conf - Specifies platform integration function * pointers for DDK events such as device init and term. @@ -103,57 +95,6 @@ struct kbase_platform_funcs_conf { * can be accessed (and possibly terminated) in here. */ void (*platform_late_term_func)(struct kbase_device *kbdev); - -#if !MALI_USE_CSF - /** - * @platform_handler_context_init_func: platform specific handler for - * when a new kbase_context is created. - * @kctx - kbase_context pointer - * - * Returns 0 on success, negative error code otherwise. - * - * Function pointer for platform specific initialization of a kernel - * context or NULL if not required. Called at the last stage of kernel - * context initialization. - */ - int (*platform_handler_context_init_func)(struct kbase_context *kctx); - /** - * @platform_handler_context_term_func: platform specific handler for - * when a kbase_context is terminated. - * @kctx - kbase_context pointer - * - * Function pointer for platform specific termination of a kernel - * context or NULL if not required. Called at the first stage of kernel - * context termination. - */ - void (*platform_handler_context_term_func)(struct kbase_context *kctx); - /** - * @platform_handler_atom_submit_func: platform specific handler for - * when a kbase_jd_atom is submitted. - * @katom - kbase_jd_atom pointer - * - * Function pointer for platform specific handling at the point when an - * atom is submitted to the GPU or set to NULL if not required. The - * function cannot assume that it is running in a process context. - * - * Context: The caller must hold the hwaccess_lock. Function must be - * runnable in an interrupt context. - */ - void (*platform_handler_atom_submit_func)(struct kbase_jd_atom *katom); - /** - * @platform_handler_atom_complete_func: platform specific handler for - * when a kbase_jd_atom completes. - * @katom - kbase_jd_atom pointer - * - * Function pointer for platform specific handling at the point when an - * atom stops running on the GPU or set to NULL if not required. The - * function cannot assume that it is running in a process context. - * - * Context: The caller must hold the hwaccess_lock. Function must be - * runnable in an interrupt context. - */ - void (*platform_handler_atom_complete_func)(struct kbase_jd_atom *katom); -#endif }; /* @@ -232,7 +173,7 @@ struct kbase_pm_callback_conf { * to the OS from within this function. * The runtime calls can be triggered by calls from @ref power_off_callback * and @ref power_on_callback. - * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * Note: for linux the kernel must have CONFIG_PM enabled to use this feature. * * @return 0 on success, else int error code. */ @@ -243,21 +184,21 @@ struct kbase_pm_callback_conf { * The runtime power management callbacks @ref power_runtime_off_callback * and @ref power_runtime_on_callback should no longer be called by the * OS on completion of this function. - * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * Note: for linux the kernel must have CONFIG_PM enabled to use this feature. */ void (*power_runtime_term_callback)(struct kbase_device *kbdev); /** Callback for runtime power-off power management callback * * For linux this callback will be called by the kernel runtime_suspend callback. - * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * Note: for linux the kernel must have CONFIG_PM enabled to use this feature. */ void (*power_runtime_off_callback)(struct kbase_device *kbdev); /** Callback for runtime power-on power management callback * * For linux this callback will be called by the kernel runtime_resume callback. - * Note: for linux the kernel must have CONFIG_PM_RUNTIME enabled to use this feature. + * Note: for linux the kernel must have CONFIG_PM enabled to use this feature. * * @return 0 on success, else OS error code. */ @@ -281,7 +222,7 @@ struct kbase_pm_callback_conf { * This callback is optional and if not provided regular autosuspend * will be triggered. * - * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use + * Note: The Linux kernel must have CONFIG_PM enabled to use * this feature. * * Return 0 if GPU can be suspended, positive value if it can not be @@ -316,7 +257,7 @@ struct kbase_pm_callback_conf { * GPU registers still remain accessible until @power_off_callback gets * invoked later on the expiry of auto-suspend timer. * - * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use + * Note: The Linux kernel must have CONFIG_PM enabled to use * this feature. */ void (*power_runtime_gpu_idle_callback)(struct kbase_device *kbdev); @@ -339,7 +280,7 @@ struct kbase_pm_callback_conf { * to avoid a potential deadlock due to the runtime suspend happening * simultaneously from some other thread. * - * Note: The Linux kernel must have CONFIG_PM_RUNTIME enabled to use + * Note: The Linux kernel must have CONFIG_PM enabled to use * this feature. */ void (*power_runtime_gpu_active_callback)(struct kbase_device *kbdev); @@ -514,56 +455,6 @@ int kbasep_platform_device_late_init(struct kbase_device *kbdev); */ void kbasep_platform_device_late_term(struct kbase_device *kbdev); -#if !MALI_USE_CSF -/** - * kbasep_platform_context_init - Platform specific callback when a kernel - * context is created - * @kctx: kbase_context pointer - * - * Function calls a platform defined routine if specified in the configuration - * attributes. The routine can initialize any per kernel context structures - * that are required for the GPU block to function. - * - * Return: 0 if no errors were encountered. Negative error code otherwise. - */ -int kbasep_platform_context_init(struct kbase_context *kctx); - -/** - * kbasep_platform_context_term - Platform specific callback when a kernel - * context is terminated - * @kctx: kbase_context pointer - * - * Function calls a platform defined routine if specified in the configuration - * attributes. The routine should terminate any per kernel context structures - * created as part of &kbasep_platform_context_init. - * - */ -void kbasep_platform_context_term(struct kbase_context *kctx); - -/** - * kbasep_platform_event_atom_submit - Platform specific callback when an atom - * is submitted to the GPU - * @katom: kbase_jd_atom pointer - * - * Function calls a platform defined routine if specified in the configuration - * attributes. The routine should not assume that it is in a process context. - * - * Return: 0 if no errors were encountered. Negative error code otherwise. - */ -void kbasep_platform_event_atom_submit(struct kbase_jd_atom *katom); - -/** - * kbasep_platform_event_atom_complete - Platform specific callback when an atom - * has stopped running on the GPU - * @katom: kbase_jd_atom pointer - * - * Function calls a platform defined routine if specified in the configuration - * attributes. The routine should not assume that it is in a process context. - * - */ -void kbasep_platform_event_atom_complete(struct kbase_jd_atom *katom); -#endif - #ifndef CONFIG_OF /** * kbase_platform_register - Register a platform device for the GPU diff --git a/drivers/gpu/arm/valhall/mali_kbase_config_defaults.h b/drivers/gpu/arm/valhall/mali_kbase_config_defaults.h index 06298c5d1509..76b2ce2c6895 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_config_defaults.h +++ b/drivers/gpu/arm/valhall/mali_kbase_config_defaults.h @@ -314,13 +314,6 @@ enum { /* MAX allowed timeout value(ms) on host side, should be less than ANR timeout */ #define MAX_TIMEOUT_MS (4500) -#if !MALI_USE_CSF -/* Default number of milliseconds given for other jobs on the GPU to be - * soft-stopped when the GPU needs to be reset. - */ -#define JM_DEFAULT_RESET_TIMEOUT_MS (1) /* 1 ms */ -#endif /* !MALI_USE_CSF */ - /* Timeout for polling the GPU PRFCNT_ACTIVE bit in clock cycles. * * Based on 120s timeout at 100MHz, based on original MAX_LOOPS value. diff --git a/drivers/gpu/arm/valhall/mali_kbase_core_linux.c b/drivers/gpu/arm/valhall/mali_kbase_core_linux.c index bc7c487611e9..081752a4bf60 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/valhall/mali_kbase_core_linux.c @@ -41,28 +41,20 @@ #include "mali_kbase_debugfs_helper.h" #include #include -#if !MALI_USE_CSF -#include -#endif /* !MALI_USE_CSF */ #ifdef CONFIG_MALI_VALHALL_PRFCNT_SET_SELECT_VIA_DEBUG_FS #include #endif #include #include -#if !MALI_USE_CSF -#include "mali_kbase_kinstr_jm.h" -#endif #include "hwcnt/mali_kbase_hwcnt_context.h" #include "hwcnt/mali_kbase_hwcnt_virtualizer.h" #include "mali_kbase_kinstr_prfcnt.h" -#if MALI_USE_CSF #include "csf/mali_kbase_csf_firmware.h" #include "csf/mali_kbase_csf_tiler_heap.h" #include "csf/mali_kbase_csf_csg_debugfs.h" #include "csf/mali_kbase_csf_cpu_queue.h" #include "csf/mali_kbase_csf_event.h" #include "csf/mali_kbase_csf_ne_debugfs.h" -#endif #include "arbiter/mali_kbase_arbiter_pm.h" #include "mali_kbase_cs_experimental.h" @@ -150,7 +142,6 @@ struct mali_kbase_capability_def { * in the table. Less efficient but potentially safer. */ static const struct mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CAPS] = { -#if MALI_USE_CSF { 1, 0 }, /* SYSTEM_MONITOR */ { 1, 0 }, /* JIT_PRESSURE_LIMIT */ { 1, 22 }, /* QUERY_MEM_DONT_NEED */ @@ -169,26 +160,6 @@ static const struct mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CA { 1, 32 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_7 */ { U16_MAX, U16_MAX }, /* REJECT_ALLOC_MEM_UNUSED_BIT_29 */ { 1, 35 } /* CSG_CS_USER_PAGE_ALLOCATION */ -#else - { 11, 15 }, /* SYSTEM_MONITOR */ - { 11, 25 }, /* JIT_PRESSURE_LIMIT */ - { 11, 40 }, /* QUERY_MEM_DONT_NEED */ - { 11, 2 }, /* QUERY_MEM_GROW_ON_GPF */ - { 11, 2 }, /* QUERY_MEM_PROTECTED */ - { 11, 43 }, /* QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP */ - { 11, 43 }, /* QUERY_MEM_KERNEL_SYNC */ - { 11, 44 }, /* QUERY_MEM_SAME_VA */ - { 11, 46 }, /* REJECT_ALLOC_MEM_DONT_NEED */ - { 11, 46 }, /* REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS */ - { 11, 46 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_8 */ - { 11, 46 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_19 */ - { U16_MAX, U16_MAX }, /* REJECT_ALLOC_MEM_UNUSED_BIT_20 */ - { 11, 48 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_27 */ - { 11, 48 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_5 */ - { 11, 48 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_7 */ - { 11, 48 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_29 */ - { U16_MAX, U16_MAX } /* CSG_CS_USER_PAGE_ALLOCATION */ -#endif }; #if (KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) @@ -496,7 +467,6 @@ static int get_irqs(struct kbase_device *kbdev, struct platform_device *pdev) return 0; } -#if MALI_USE_CSF /** * get_irq_irqaw - Get interrupt information from the device tree. * @@ -531,7 +501,6 @@ static int get_irq_irqaw(struct kbase_device *kbdev, struct platform_device *pde return 0; } -#endif /* MALI_USE_CSF */ int kbase_get_irqs(struct kbase_device *kbdev) { @@ -543,7 +512,6 @@ int kbase_get_irqs(struct kbase_device *kbdev) if (!result) return result; -#if MALI_USE_CSF /* return error if any one of the GPU, JOB, MMU * interrupts missing. Don't lookup for IRQAW. */ @@ -553,7 +521,6 @@ int kbase_get_irqs(struct kbase_device *kbdev) } result = get_irq_irqaw(kbdev, pdev); -#endif /* MALI_USE_CSF */ if (result) dev_err(kbdev->dev, "Invalid or No interrupt resources"); @@ -811,11 +778,6 @@ static int kbase_api_set_flags(struct kbase_file *kfile, struct kbase_ioctl_set_ if (mali_kbase_supports_system_monitor(api_version)) { err = kbase_file_create_kctx(kfile, flags->create_flags); } else { -#if !MALI_USE_CSF - struct kbasep_js_kctx_info *js_kctx_info = NULL; - unsigned long irq_flags = 0; -#endif - /* If setup is incomplete (e.g. because the API version * wasn't set) then we have to give up. */ @@ -823,35 +785,15 @@ static int kbase_api_set_flags(struct kbase_file *kfile, struct kbase_ioctl_set_ if (unlikely(!kctx)) return -EPERM; -#if MALI_USE_CSF - /* On CSF GPUs Job Manager interface isn't used to submit jobs + /* On CSF GPUs Job Manager interface isn't used to submit jobs * (there are no job slots). So the legacy job manager path to * submit jobs needs to remain disabled for CSF GPUs. */ -#else - js_kctx_info = &kctx->jctx.sched_info; - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); - /* Translate the flags */ - if ((flags->create_flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0) - kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED); - - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -#endif } return err; } -#if !MALI_USE_CSF -static int kbase_api_job_submit(struct kbase_context *kctx, struct kbase_ioctl_job_submit *submit) -{ - return kbase_jd_submit(kctx, u64_to_user_ptr(submit->addr), submit->nr_atoms, - submit->stride, false); -} -#endif /* !MALI_USE_CSF */ - static int kbase_api_get_gpuprops(struct kbase_file *kfile, struct kbase_ioctl_get_gpuprops *get_props) { @@ -875,15 +817,6 @@ static int kbase_api_get_gpuprops(struct kbase_file *kfile, return (int)kprops->prop_buffer_size; } -#if !MALI_USE_CSF -static int kbase_api_post_term(struct kbase_context *kctx) -{ - kbase_event_close(kctx); - return 0; -} -#endif /* !MALI_USE_CSF */ - -#if MALI_USE_CSF static int kbase_api_mem_alloc_ex(struct kbase_context *kctx, union kbase_ioctl_mem_alloc_ex *alloc_ex) { @@ -994,46 +927,6 @@ static int kbase_api_mem_alloc(struct kbase_context *kctx, union kbase_ioctl_mem return ret; } -#else -static int kbase_api_mem_alloc(struct kbase_context *kctx, union kbase_ioctl_mem_alloc *alloc) -{ - struct kbase_va_region *reg; - base_mem_alloc_flags flags = alloc->in.flags; - u64 gpu_va; - - /* Calls to this function are inherently asynchronous, with respect to - * MMU operations. - */ - const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC; - - if (!kbase_mem_allow_alloc(kctx)) - return -EINVAL; - - if (flags & BASE_MEM_FLAGS_KERNEL_ONLY) - return -ENOMEM; - - /* Force SAME_VA if a 64-bit client. - * The only exception is GPU-executable memory if an EXEC_VA zone - * has been initialized. In that case, GPU-executable memory may - * or may not be SAME_VA. - */ - if ((!kbase_ctx_flag(kctx, KCTX_COMPAT)) && kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA)) { - if (!(flags & BASE_MEM_PROT_GPU_EX) || !kbase_has_exec_va_zone(kctx)) - flags |= BASE_MEM_SAME_VA; - } - - reg = kbase_mem_alloc(kctx, alloc->in.va_pages, alloc->in.commit_pages, alloc->in.extension, - &flags, &gpu_va, mmu_sync_info); - - if (!reg) - return -ENOMEM; - - alloc->out.flags = flags; - alloc->out.gpu_va = gpu_va; - - return 0; -} -#endif /* MALI_USE_CSF */ static int kbase_api_mem_query(struct kbase_context *kctx, union kbase_ioctl_mem_query *query) { @@ -1045,13 +938,6 @@ static int kbase_api_mem_free(struct kbase_context *kctx, struct kbase_ioctl_mem return kbase_mem_free(kctx, free->gpu_addr); } -#if !MALI_USE_CSF -static int kbase_api_kinstr_jm_fd(struct kbase_context *kctx, union kbase_kinstr_jm_fd *arg) -{ - return kbase_kinstr_jm_get_fd(kctx->kinstr_jm, arg); -} -#endif - static int kbase_api_get_cpu_gpu_timeinfo(struct kbase_context *kctx, union kbase_ioctl_get_cpu_gpu_timeinfo *timeinfo) { @@ -1060,6 +946,9 @@ static int kbase_api_get_cpu_gpu_timeinfo(struct kbase_context *kctx, u64 timestamp = 0; u64 cycle_cnt = 0; + if (kbase_io_is_aw_removed(kctx->kbdev)) + return -ENODEV; + kbase_pm_context_active(kctx->kbdev); kbase_backend_get_gpu_time(kctx->kbdev, @@ -1137,7 +1026,7 @@ static int kbase_api_mem_exec_init(struct kbase_context *kctx, static int kbase_api_mem_sync(struct kbase_context *kctx, struct kbase_ioctl_mem_sync *sync) { - struct basep_syncset sset = { .mem_handle.basep.handle = sync->handle, + struct basep_syncset sset = { .gpu_va = sync->gpu_va, .user_addr = sync->user_addr, .size = sync->size, .type = sync->type }; @@ -1327,17 +1216,6 @@ static int kbase_api_mem_profile_add(struct kbase_context *kctx, return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); } -#if !MALI_USE_CSF -static int kbase_api_soft_event_update(struct kbase_context *kctx, - struct kbase_ioctl_soft_event_update *update) -{ - if (update->flags != 0) - return -EINVAL; - - return kbase_soft_event_update(kctx, update->event, update->new_status); -} -#endif /* !MALI_USE_CSF */ - static int kbase_api_sticky_resource_map(struct kbase_context *kctx, struct kbase_ioctl_sticky_resource_map *map) { @@ -1426,7 +1304,6 @@ static int kbase_api_tlstream_stats(struct kbase_context *kctx, } #endif /* MALI_UNIT_TEST */ -#if MALI_USE_CSF static int kbasep_cs_event_signal(struct kbase_context *kctx) { kbase_csf_event_signal_notify_gpu(kctx); @@ -1703,8 +1580,16 @@ static int kbasep_ioctl_cs_cpu_queue_dump(struct kbase_context *kctx, return kbase_csf_cpu_queue_dump_buffer(kctx, cpu_queue_info->buffer, cpu_queue_info->size); } +#if MALI_UNIT_TEST +int kbase_ioctl_read_user_page(struct kbase_context *kctx, + union kbase_ioctl_read_user_page *user_page); + +int kbase_ioctl_read_user_page(struct kbase_context *kctx, + union kbase_ioctl_read_user_page *user_page) +#else static int kbase_ioctl_read_user_page(struct kbase_context *kctx, union kbase_ioctl_read_user_page *user_page) +#endif { struct kbase_device *kbdev = kctx->kbdev; unsigned long flags; @@ -1714,7 +1599,7 @@ static int kbase_ioctl_read_user_page(struct kbase_context *kctx, return -EINVAL; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (!kbase_io_is_gpu_powered(kbdev) || kbase_io_is_aw_removed(kbdev)) + if (!kbase_io_has_gpu(kbdev)) user_page->out.val_lo = POWER_DOWN_LATEST_FLUSH_VALUE; else user_page->out.val_lo = kbase_reg_read32(kbdev, USER_ENUM(LATEST_FLUSH)); @@ -1723,19 +1608,15 @@ static int kbase_ioctl_read_user_page(struct kbase_context *kctx, return 0; } -#endif /* MALI_USE_CSF */ +#if MALI_UNIT_TEST +KBASE_EXPORT_TEST_API(kbase_ioctl_read_user_page); +#endif static int kbasep_ioctl_context_priority_check(struct kbase_context *kctx, struct kbase_ioctl_context_priority_check *priority_check) { -#if MALI_USE_CSF priority_check->priority = kbase_csf_priority_check(kctx->kbdev, priority_check->priority); -#else - base_jd_prio req_priority = (base_jd_prio)priority_check->priority; - - priority_check->priority = (u8)kbase_js_priority_check(kctx->kbdev, req_priority); -#endif return 0; } @@ -1811,27 +1692,14 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) /* Normal ioctls */ switch (cmd) { -#if !MALI_USE_CSF - case KBASE_IOCTL_JOB_SUBMIT: - KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, kbase_api_job_submit, - struct kbase_ioctl_job_submit, kctx); - break; -#endif /* !MALI_USE_CSF */ -#if !MALI_USE_CSF - case KBASE_IOCTL_POST_TERM: - KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, kbase_api_post_term, kctx); - break; -#endif /* !MALI_USE_CSF */ case KBASE_IOCTL_MEM_ALLOC: KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, kbase_api_mem_alloc, union kbase_ioctl_mem_alloc, kctx); break; -#if MALI_USE_CSF case KBASE_IOCTL_MEM_ALLOC_EX: KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC_EX, kbase_api_mem_alloc_ex, union kbase_ioctl_mem_alloc_ex, kctx); break; -#endif case KBASE_IOCTL_MEM_QUERY: KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_QUERY, kbase_api_mem_query, union kbase_ioctl_mem_query, kctx); @@ -1910,13 +1778,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) struct kbase_ioctl_mem_profile_add, kctx); break; -#if !MALI_USE_CSF - case KBASE_IOCTL_SOFT_EVENT_UPDATE: - KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, kbase_api_soft_event_update, - struct kbase_ioctl_soft_event_update, kctx); - break; -#endif /* !MALI_USE_CSF */ - case KBASE_IOCTL_STICKY_RESOURCE_MAP: KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STICKY_RESOURCE_MAP, kbase_api_sticky_resource_map, @@ -1929,12 +1790,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) break; /* Instrumentation. */ -#if !MALI_USE_CSF - case KBASE_IOCTL_KINSTR_JM_FD: - KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_KINSTR_JM_FD, kbase_api_kinstr_jm_fd, - union kbase_kinstr_jm_fd, kctx); - break; -#endif case KBASE_IOCTL_GET_CPU_GPU_TIMEINFO: KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_GET_CPU_GPU_TIMEINFO, kbase_api_get_cpu_gpu_timeinfo, @@ -1946,7 +1801,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) struct kbase_ioctl_hwcnt_values, kctx); break; #endif /* CONFIG_MALI_VALHALL_NO_MALI */ -#if MALI_USE_CSF case KBASE_IOCTL_CS_EVENT_SIGNAL: KBASE_HANDLE_IOCTL(KBASE_IOCTL_CS_EVENT_SIGNAL, kbasep_cs_event_signal, kctx); break; @@ -2042,7 +1896,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_READ_USER_PAGE, kbase_ioctl_read_user_page, union kbase_ioctl_read_user_page, kctx); break; -#endif /* MALI_USE_CSF */ #if MALI_UNIT_TEST case KBASE_IOCTL_TLSTREAM_STATS: KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, kbase_api_tlstream_stats, @@ -2066,7 +1919,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return -ENOIOCTLCMD; } -#if MALI_USE_CSF static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) { struct kbase_file *const kfile = filp->private_data; @@ -2110,54 +1962,6 @@ static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, lof return data_size; } -#else /* MALI_USE_CSF */ -static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) -{ - struct kbase_file *const kfile = filp->private_data; - struct kbase_context *const kctx = kbase_file_get_kctx_if_setup_complete(kfile); - struct base_jd_event_v2 uevent; - int out_count = 0; - - CSTD_UNUSED(f_pos); - - if (unlikely(!kctx)) - return -EPERM; - - if (count < sizeof(uevent)) - return -ENOBUFS; - - memset(&uevent, 0, sizeof(uevent)); - - do { - while (kbase_event_dequeue(kctx, &uevent)) { - if (out_count > 0) - goto out; - - if (filp->f_flags & O_NONBLOCK) - return -EAGAIN; - - if (wait_event_interruptible(kctx->event_queue, - kbase_event_pending(kctx)) != 0) - return -ERESTARTSYS; - } - if (uevent.event_code == BASE_JD_EVENT_DRV_TERMINATED) { - if (out_count == 0) - return -EPIPE; - goto out; - } - - if (copy_to_user(buf, &uevent, sizeof(uevent)) != 0) - return -EFAULT; - - buf += sizeof(uevent); - out_count++; - count -= sizeof(uevent); - } while (count >= sizeof(uevent)); - -out: - return out_count * sizeof(uevent); -} -#endif /* MALI_USE_CSF */ static __poll_t kbase_poll(struct file *filp, poll_table *wait) { @@ -2193,7 +1997,6 @@ void kbase_event_wakeup(struct kbase_context *kctx) KBASE_EXPORT_TEST_API(kbase_event_wakeup); -#if MALI_USE_CSF int kbase_event_pending(struct kbase_context *kctx) { KBASE_DEBUG_ASSERT(kctx); @@ -2204,17 +2007,6 @@ int kbase_event_pending(struct kbase_context *kctx) return (atomic_read(&kctx->event_count) != 0) || kbase_csf_event_error_pending(kctx) || kbase_csf_cpu_queue_dump_needed(kctx); } -#else -int kbase_event_pending(struct kbase_context *kctx) -{ - KBASE_DEBUG_ASSERT(kctx); - - if (unlikely(!kctx)) - return -EPERM; - - return (atomic_read(&kctx->event_count) != 0) || (atomic_read(&kctx->event_closed) != 0); -} -#endif KBASE_EXPORT_TEST_API(kbase_event_pending); @@ -2396,10 +2188,9 @@ static ssize_t core_mask_show(struct device *dev, struct device_attribute *attr, { struct kbase_device *kbdev; unsigned long flags; + u64 debug_mask; + u64 ca_mask; ssize_t ret = 0; -#if !MALI_USE_CSF - size_t i; -#endif CSTD_UNUSED(attr); @@ -2410,35 +2201,30 @@ static ssize_t core_mask_show(struct device *dev, struct device_attribute *attr, spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current debug core mask : 0x%llX\n", - kbdev->pm.debug_core_mask); - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), - "Current desired core mask : 0x%llX\n", kbase_pm_ca_get_core_mask(kbdev)); - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), - "Current in use core mask : 0x%llX\n", kbdev->pm.backend.shaders_avail); -#else - for (i = 0; i < BASE_JM_MAX_NR_SLOTS; i++) { - if (PAGE_SIZE < ret) - goto out_unlock; - - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), - "Current core mask (JS%zu) : 0x%llX\n", i, - kbdev->pm.debug_core_mask[i]); + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + ca_mask = kbase_pm_ca_get_gov_core_mask(kbdev); + debug_mask = kbase_pm_ca_get_sysfs_gov_core_mask(kbdev); + } else { + ca_mask = kbase_pm_ca_get_core_mask(kbdev); + debug_mask = kbase_pm_ca_get_debug_core_mask(kbdev); } -#endif /* MALI_USE_CSF */ + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current debug core mask : 0x%llX\n", + debug_mask); + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), + "Current desired core mask : 0x%llX\n", ca_mask); + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), + "Current in use core mask : 0x%llX\n", + kbdev->pm.backend.shaders_avail); ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Available core mask : 0x%llX\n", kbdev->gpu_props.shader_present); -#if !MALI_USE_CSF -out_unlock: -#endif + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return ret; } -#if MALI_USE_CSF struct kbase_core_mask { u64 new_core_mask; }; @@ -2458,6 +2244,8 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con { u64 new_core_mask = new_mask->new_core_mask; u64 shader_present; + u64 ca_mask; + u64 debug_mask; unsigned long flags; int ret = 0; @@ -2465,6 +2253,14 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con kbase_pm_lock(kbdev); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT)) { + ca_mask = kbdev->pm.backend.ca_gov_cores_enabled; + debug_mask = kbase_pm_ca_get_sysfs_gov_core_mask(kbdev); + } else { + ca_mask = kbdev->pm.backend.ca_cores_enabled; + debug_mask = kbase_pm_ca_get_debug_core_mask(kbdev); + } + shader_present = kbdev->gpu_props.shader_present; if ((new_core_mask & shader_present) != new_core_mask) { @@ -2473,13 +2269,16 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con new_core_mask, shader_present); ret = -EINVAL; goto exit; - } else if (!(new_core_mask & shader_present & kbdev->pm.backend.ca_cores_enabled)) { - dev_err(kbdev->dev, - "Invalid requested core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", - new_core_mask, kbdev->gpu_props.shader_present, - kbdev->pm.backend.ca_cores_enabled); - ret = -EINVAL; - goto exit; + } else if (!(new_core_mask & shader_present & ca_mask)) { + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT) || + (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GOV_CORE_MASK_SUPPORT) && + new_core_mask != 0)) { + dev_err(kbdev->dev, + "Invalid requested core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", + new_core_mask, kbdev->gpu_props.shader_present, ca_mask); + ret = -EINVAL; + goto exit; + } } if (kbase_csf_dev_has_ne(kbdev)) { @@ -2502,7 +2301,7 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con } } - if (kbdev->pm.debug_core_mask != new_core_mask) + if (debug_mask != new_core_mask) kbase_pm_set_debug_core_mask(kbdev, new_core_mask); exit: @@ -2512,97 +2311,6 @@ exit: return ret; } -#else -struct kbase_core_mask { - u64 new_core_mask[BASE_JM_MAX_NR_SLOTS]; -}; - -static int core_mask_parse(struct kbase_device *const kbdev, const char *const buf, - struct kbase_core_mask *const mask) -{ - int items; - - items = sscanf(buf, "%llx %llx %llx", &mask->new_core_mask[0], &mask->new_core_mask[1], - &mask->new_core_mask[2]); - - if (items != 1 && items != BASE_JM_MAX_NR_SLOTS) { - dev_err(kbdev->dev, "Couldn't process core mask write operation.\n" - "Use format \n" - "or \n"); - return -EINVAL; - } - - /* If only one value was provided, set all other core masks equal to the value. */ - if (items == 1) { - size_t i; - - for (i = 1; i < BASE_JM_MAX_NR_SLOTS; i++) - mask->new_core_mask[i] = mask->new_core_mask[0]; - } - - return 0; -} - -static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *const new_mask) -{ - u64 shader_present = kbdev->gpu_props.shader_present; - u64 group_core_mask = kbdev->gpu_props.coherency_info.group.core_mask; - u64 *new_core_mask; - unsigned long flags; - int ret = 0; - size_t i; - - kbase_pm_lock(kbdev); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - new_core_mask = &new_mask->new_core_mask[0]; - - for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) { - if ((new_core_mask[i] & shader_present) != new_core_mask[i]) { - dev_err(kbdev->dev, - "Invalid core mask 0x%llX for JS %zu: Includes non-existent cores (present = 0x%llX)", - new_core_mask[i], i, shader_present); - ret = -EINVAL; - goto exit; - - } else if (!(new_core_mask[i] & shader_present & - kbdev->pm.backend.ca_cores_enabled)) { - dev_err(kbdev->dev, - "Invalid core mask 0x%llX for JS %zu: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", - new_core_mask[i], i, kbdev->gpu_props.shader_present, - kbdev->pm.backend.ca_cores_enabled); - ret = -EINVAL; - goto exit; - } else if (!(new_core_mask[i] & group_core_mask)) { - dev_err(kbdev->dev, - "Invalid core mask 0x%llX for JS %zu: No intersection with group 0 core mask 0x%llX", - new_core_mask[i], i, group_core_mask); - ret = -EINVAL; - goto exit; - } else if (!(new_core_mask[i] & kbdev->gpu_props.curr_config.shader_present)) { - dev_err(kbdev->dev, - "Invalid core mask 0x%llX for JS %zu: No intersection with current core mask 0x%llX", - new_core_mask[i], i, kbdev->gpu_props.curr_config.shader_present); - ret = -EINVAL; - goto exit; - } - } - - for (i = 0; i < BASE_JM_MAX_NR_SLOTS; i++) { - if (kbdev->pm.debug_core_mask[i] != new_core_mask[i]) { - kbase_pm_set_debug_core_mask(kbdev, new_core_mask, BASE_JM_MAX_NR_SLOTS); - break; - } - } - -exit: - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - kbase_pm_unlock(kbdev); - - return ret; -} - -#endif /** * core_mask_store - Store callback for the core_mask sysfs file. @@ -2652,446 +2360,6 @@ static ssize_t core_mask_store(struct device *dev, struct device_attribute *attr */ static DEVICE_ATTR_RW(core_mask); -#if !MALI_USE_CSF -/** - * soft_job_timeout_store - Store callback for the soft_job_timeout sysfs - * file. - * - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. - * @count: The number of bytes to write to the sysfs file. - * - * This allows setting the timeout for software jobs. Waiting soft event wait - * jobs will be cancelled after this period expires, while soft fence wait jobs - * will print debug information if the fence debug feature is enabled. - * - * This is expressed in milliseconds. - * - * Return: count if the function succeeded. An error code on failure. - */ -static ssize_t soft_job_timeout_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *kbdev; - int soft_job_timeout_ms; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - if ((kstrtoint(buf, 0, &soft_job_timeout_ms) != 0) || (soft_job_timeout_ms <= 0)) - return -EINVAL; - - atomic_set(&kbdev->js_data.soft_job_timeout_ms, soft_job_timeout_ms); - - return (ssize_t)count; -} - -/** - * soft_job_timeout_show - Show callback for the soft_job_timeout sysfs - * file. - * - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer for the sysfs file contents. - * - * This will return the timeout for the software jobs. - * - * Return: The number of bytes output to buf. - */ -static ssize_t soft_job_timeout_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *kbdev; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - return scnprintf(buf, PAGE_SIZE, "%i\n", atomic_read(&kbdev->js_data.soft_job_timeout_ms)); -} - -static DEVICE_ATTR_RW(soft_job_timeout); - -static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, int default_ticks, - u32 old_ticks) -{ - if (timeout_ms > 0) { - u64 ticks = (u64)timeout_ms * 1000000ULL; - - do_div(ticks, kbdev->js_data.scheduling_period_ns); - if (!ticks) - return 1; - return ticks; - } else if (timeout_ms < 0) { - return (u32)default_ticks; - } else { - return old_ticks; - } -} - -/** - * js_timeouts_store - Store callback for the js_timeouts sysfs file. - * - * @dev: The device with sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file - * @count: The number of bytes to write to the sysfs file - * - * This function is called to get the contents of the js_timeouts sysfs - * file. This file contains five values separated by whitespace. The values - * are basically the same as %JS_SOFT_STOP_TICKS, %JS_HARD_STOP_TICKS_SS, - * %JS_HARD_STOP_TICKS_DUMPING, %JS_RESET_TICKS_SS, %JS_RESET_TICKS_DUMPING - * configuration values (in that order), with the difference that the js_timeout - * values are expressed in MILLISECONDS. - * - * The js_timeouts sysfile file allows the current values in - * use by the job scheduler to get override. Note that a value needs to - * be other than 0 for it to override the current job scheduler value. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t js_timeouts_store(struct device *dev, struct device_attribute *attr, const char *buf, - size_t count) -{ - struct kbase_device *kbdev; - int items; - long js_soft_stop_ms; - long js_soft_stop_ms_cl; - long js_hard_stop_ms_ss; - long js_hard_stop_ms_cl; - long js_hard_stop_ms_dumping; - long js_reset_ms_ss; - long js_reset_ms_cl; - long js_reset_ms_dumping; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - items = sscanf(buf, "%ld %ld %ld %ld %ld %ld %ld %ld", &js_soft_stop_ms, - &js_soft_stop_ms_cl, &js_hard_stop_ms_ss, &js_hard_stop_ms_cl, - &js_hard_stop_ms_dumping, &js_reset_ms_ss, &js_reset_ms_cl, - &js_reset_ms_dumping); - - if (items == 8) { - struct kbasep_js_device_data *js_data = &kbdev->js_data; - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - -#define UPDATE_TIMEOUT(ticks_name, ms_name, default) \ - do { \ - js_data->ticks_name = \ - timeout_ms_to_ticks(kbdev, ms_name, default, js_data->ticks_name); \ - dev_dbg(kbdev->dev, "Overriding " #ticks_name " with %lu ticks (%lu ms)\n", \ - (unsigned long)js_data->ticks_name, ms_name); \ - } while (0) - - UPDATE_TIMEOUT(soft_stop_ticks, js_soft_stop_ms, DEFAULT_JS_SOFT_STOP_TICKS); - UPDATE_TIMEOUT(soft_stop_ticks_cl, js_soft_stop_ms_cl, - DEFAULT_JS_SOFT_STOP_TICKS_CL); - UPDATE_TIMEOUT(hard_stop_ticks_ss, js_hard_stop_ms_ss, - DEFAULT_JS_HARD_STOP_TICKS_SS); - UPDATE_TIMEOUT(hard_stop_ticks_cl, js_hard_stop_ms_cl, - DEFAULT_JS_HARD_STOP_TICKS_CL); - UPDATE_TIMEOUT(hard_stop_ticks_dumping, js_hard_stop_ms_dumping, - DEFAULT_JS_HARD_STOP_TICKS_DUMPING); - UPDATE_TIMEOUT(gpu_reset_ticks_ss, js_reset_ms_ss, DEFAULT_JS_RESET_TICKS_SS); - UPDATE_TIMEOUT(gpu_reset_ticks_cl, js_reset_ms_cl, DEFAULT_JS_RESET_TICKS_CL); - UPDATE_TIMEOUT(gpu_reset_ticks_dumping, js_reset_ms_dumping, - DEFAULT_JS_RESET_TICKS_DUMPING); - - kbase_js_set_timeouts(kbdev); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return (ssize_t)count; - } - - dev_err(kbdev->dev, - "Couldn't process js_timeouts write operation.\n" - "Use format \n" - "Write 0 for no change, -1 to restore default timeout\n"); - return -EINVAL; -} - -static unsigned long get_js_timeout_in_ms(u32 scheduling_period_ns, u32 ticks) -{ - u64 ms = (u64)ticks * scheduling_period_ns; - - do_div(ms, 1000000UL); - return ms; -} - -/** - * js_timeouts_show - Show callback for the js_timeouts sysfs file. - * - * @dev: The device this sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The output buffer for the sysfs file contents - * - * This function is called to get the contents of the js_timeouts sysfs - * file. It returns the last set values written to the js_timeouts sysfs file. - * If the file didn't get written yet, the values will be current setting in - * use. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t js_timeouts_show(struct device *dev, struct device_attribute *attr, char *const buf) -{ - struct kbase_device *kbdev; - ssize_t ret; - unsigned long js_soft_stop_ms; - unsigned long js_soft_stop_ms_cl; - unsigned long js_hard_stop_ms_ss; - unsigned long js_hard_stop_ms_cl; - unsigned long js_hard_stop_ms_dumping; - unsigned long js_reset_ms_ss; - unsigned long js_reset_ms_cl; - unsigned long js_reset_ms_dumping; - u32 scheduling_period_ns; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - scheduling_period_ns = kbdev->js_data.scheduling_period_ns; - -#define GET_TIMEOUT(name) get_js_timeout_in_ms(scheduling_period_ns, kbdev->js_data.name) - - js_soft_stop_ms = GET_TIMEOUT(soft_stop_ticks); - js_soft_stop_ms_cl = GET_TIMEOUT(soft_stop_ticks_cl); - js_hard_stop_ms_ss = GET_TIMEOUT(hard_stop_ticks_ss); - js_hard_stop_ms_cl = GET_TIMEOUT(hard_stop_ticks_cl); - js_hard_stop_ms_dumping = GET_TIMEOUT(hard_stop_ticks_dumping); - js_reset_ms_ss = GET_TIMEOUT(gpu_reset_ticks_ss); - js_reset_ms_cl = GET_TIMEOUT(gpu_reset_ticks_cl); - js_reset_ms_dumping = GET_TIMEOUT(gpu_reset_ticks_dumping); - -#undef GET_TIMEOUT - - ret = scnprintf(buf, PAGE_SIZE, "%lu %lu %lu %lu %lu %lu %lu %lu\n", js_soft_stop_ms, - js_soft_stop_ms_cl, js_hard_stop_ms_ss, js_hard_stop_ms_cl, - js_hard_stop_ms_dumping, js_reset_ms_ss, js_reset_ms_cl, - js_reset_ms_dumping); - - if (ret >= (ssize_t)PAGE_SIZE) { - buf[PAGE_SIZE - 2] = '\n'; - buf[PAGE_SIZE - 1] = '\0'; - ret = PAGE_SIZE - 1; - } - - return ret; -} - -/* - * The sysfs file js_timeouts. - * - * This is used to override the current job scheduler values for - * JS_STOP_STOP_TICKS_SS - * JS_STOP_STOP_TICKS_CL - * JS_HARD_STOP_TICKS_SS - * JS_HARD_STOP_TICKS_CL - * JS_HARD_STOP_TICKS_DUMPING - * JS_RESET_TICKS_SS - * JS_RESET_TICKS_CL - * JS_RESET_TICKS_DUMPING. - */ -static DEVICE_ATTR_RW(js_timeouts); - -static u32 get_new_js_timeout(u32 old_period, u32 old_ticks, u32 new_scheduling_period_ns) -{ - u64 ticks = (u64)old_period * (u64)old_ticks; - - do_div(ticks, new_scheduling_period_ns); - return ticks ? ticks : 1; -} - -/** - * js_scheduling_period_store - Store callback for the js_scheduling_period sysfs - * file - * @dev: The device the sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file - * @count: The number of bytes to write to the sysfs file - * - * This function is called when the js_scheduling_period sysfs file is written - * to. It checks the data written, and if valid updates the js_scheduling_period - * value - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t js_scheduling_period_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *kbdev; - int ret; - unsigned int js_scheduling_period; - u32 new_scheduling_period_ns; - u32 old_period; - struct kbasep_js_device_data *js_data; - unsigned long flags; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - js_data = &kbdev->js_data; - - ret = kstrtouint(buf, 0, &js_scheduling_period); - if (ret || !js_scheduling_period) { - dev_err(kbdev->dev, "Couldn't process js_scheduling_period write operation.\n" - "Use format \n"); - return -EINVAL; - } - - new_scheduling_period_ns = js_scheduling_period * 1000000; - - /* Update scheduling timeouts */ - mutex_lock(&js_data->runpool_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* If no contexts have been scheduled since js_timeouts was last written - * to, the new timeouts might not have been latched yet. So check if an - * update is pending and use the new values if necessary. - */ - - /* Use previous 'new' scheduling period as a base if present. */ - old_period = js_data->scheduling_period_ns; - -#define SET_TIMEOUT(name) \ - (js_data->name = \ - get_new_js_timeout(old_period, kbdev->js_data.name, new_scheduling_period_ns)) - - SET_TIMEOUT(soft_stop_ticks); - SET_TIMEOUT(soft_stop_ticks_cl); - SET_TIMEOUT(hard_stop_ticks_ss); - SET_TIMEOUT(hard_stop_ticks_cl); - SET_TIMEOUT(hard_stop_ticks_dumping); - SET_TIMEOUT(gpu_reset_ticks_ss); - SET_TIMEOUT(gpu_reset_ticks_cl); - SET_TIMEOUT(gpu_reset_ticks_dumping); - -#undef SET_TIMEOUT - - js_data->scheduling_period_ns = new_scheduling_period_ns; - - kbase_js_set_timeouts(kbdev); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_data->runpool_mutex); - - dev_dbg(kbdev->dev, "JS scheduling period: %dms\n", js_scheduling_period); - - return (ssize_t)count; -} - -/** - * js_scheduling_period_show - Show callback for the js_scheduling_period sysfs - * entry. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the GPU information. - * - * This function is called to get the current period used for the JS scheduling - * period. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t js_scheduling_period_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *kbdev; - u32 period; - ssize_t ret; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - period = kbdev->js_data.scheduling_period_ns; - - ret = scnprintf(buf, PAGE_SIZE, "%d\n", period / 1000000); - - return ret; -} - -static DEVICE_ATTR_RW(js_scheduling_period); - -#ifdef CONFIG_MALI_VALHALL_DEBUG -static ssize_t js_softstop_always_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *kbdev; - int ret; - int softstop_always; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - ret = kstrtoint(buf, 0, &softstop_always); - if (ret || ((softstop_always != 0) && (softstop_always != 1))) { - dev_err(kbdev->dev, "Couldn't process js_softstop_always write operation.\n" - "Use format \n"); - return -EINVAL; - } - - kbdev->js_data.softstop_always = (bool)softstop_always; - dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", - (kbdev->js_data.softstop_always) ? "Enabled" : "Disabled"); - return (ssize_t)count; -} - -static ssize_t js_softstop_always_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *kbdev; - ssize_t ret; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - ret = scnprintf(buf, PAGE_SIZE, "%d\n", kbdev->js_data.softstop_always); - - if (ret >= (ssize_t)PAGE_SIZE) { - buf[PAGE_SIZE - 2] = '\n'; - buf[PAGE_SIZE - 1] = '\0'; - ret = PAGE_SIZE - 1; - } - - return ret; -} - -/* - * By default, soft-stops are disabled when only a single context is present. - * The ability to enable soft-stop when only a single context is present can be - * used for debug and unit-testing purposes. - * (see CL t6xx_stress_1 unit-test as an example whereby this feature is used.) - */ -static DEVICE_ATTR_RW(js_softstop_always); -#endif /* CONFIG_MALI_VALHALL_DEBUG */ -#endif /* !MALI_USE_CSF */ - #ifdef CONFIG_MALI_VALHALL_DEBUG typedef void kbasep_debug_command_func(struct kbase_device *); @@ -3276,7 +2544,6 @@ static ssize_t gpuinfo_show(struct device *dev, struct device_attribute *attr, c } } -#if MALI_USE_CSF if (product_model == GPU_ID_PRODUCT_TTUX) { const bool rt_supported = gpu_props->gpu_features.ray_intersection; const u8 nr_cores = gpu_props->num_cores; @@ -3355,7 +2622,6 @@ static ssize_t gpuinfo_show(struct device *dev, struct device_attribute *attr, c dev_dbg(kbdev->dev, "GPU ID_Name: %s (ID: 0x%x), nr_cores(%u)\n", product_name, product_id, nr_cores); } -#endif /* MALI_USE_CSF */ return scnprintf(buf, PAGE_SIZE, "%s %d cores r%dp%d 0x%08X\n", product_name, kbdev->gpu_props.num_cores, gpu_props->gpu_id.version_major, @@ -3603,11 +2869,7 @@ static ssize_t reset_timeout_store(struct device *dev, struct device_attribute * return -EINVAL; } -#if MALI_USE_CSF default_reset_timeout = kbase_get_timeout_ms(kbdev, CSF_GPU_RESET_TIMEOUT); -#else /* MALI_USE_CSF */ - default_reset_timeout = JM_DEFAULT_RESET_TIMEOUT_MS; -#endif /* !MALI_USE_CSF */ if (reset_timeout < default_reset_timeout) dev_warn(kbdev->dev, "requested reset_timeout(%u) is smaller than default(%u)", @@ -3648,689 +2910,6 @@ static ssize_t reset_timeout_show(struct device *dev, struct device_attribute *a static DEVICE_ATTR_RW(reset_timeout); -static ssize_t mem_pool_size_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.small, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_size); -} - -static ssize_t mem_pool_size_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - ssize_t err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kbase_debugfs_helper_set_attr_from_string(buf, kbdev->mem_pools.small, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_trim); - - return err ? err : (ssize_t)count; -} - -static DEVICE_ATTR_RW(mem_pool_size); - -static ssize_t mem_pool_max_size_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.small, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_max_size); -} - -static ssize_t mem_pool_max_size_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - ssize_t err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kbase_debugfs_helper_set_attr_from_string(buf, kbdev->mem_pools.small, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_set_max_size); - - return err ? err : (ssize_t)count; -} - -static DEVICE_ATTR_RW(mem_pool_max_size); - -/** - * lp_mem_pool_size_show - Show size of the large memory pages pool. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the pool size. - * - * This function is called to get the number of large memory pages which currently populate the kbdev pool. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t lp_mem_pool_size_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.large, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_size); -} - -/** - * lp_mem_pool_size_store - Set size of the large memory pages pool. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. - * @count: The number of bytes written to the sysfs file. - * - * This function is called to set the number of large memory pages which should populate the kbdev pool. - * This may cause existing pages to be removed from the pool, or new pages to be created and then added to the pool. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t lp_mem_pool_size_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - ssize_t err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kbase_debugfs_helper_set_attr_from_string(buf, kbdev->mem_pools.large, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_trim); - - return err ? err : (ssize_t)count; -} - -static DEVICE_ATTR_RW(lp_mem_pool_size); - -/** - * lp_mem_pool_max_size_show - Show maximum size of the large memory pages pool. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the pool size. - * - * This function is called to get the maximum number of large memory pages that the kbdev pool can possibly contain. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t lp_mem_pool_max_size_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.large, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_max_size); -} - -/** - * lp_mem_pool_max_size_store - Set maximum size of the large memory pages pool. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. - * @count: The number of bytes written to the sysfs file. - * - * This function is called to set the maximum number of large memory pages that the kbdev pool can possibly contain. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t lp_mem_pool_max_size_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - ssize_t err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kbase_debugfs_helper_set_attr_from_string(buf, kbdev->mem_pools.large, - MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_debugfs_set_max_size); - - return err ? err : (ssize_t)count; -} - -static DEVICE_ATTR_RW(lp_mem_pool_max_size); - -/** - * show_simplified_mem_pool_max_size - Show the maximum size for the memory - * pool 0 of small (4KiB/16KiB/64KiB) pages. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the max size. - * - * This function is called to get the maximum size for the memory pool 0 of - * small pages. It is assumed that the maximum size value is same for - * all the pools. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t show_simplified_mem_pool_max_size(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.small, 1, - kbase_mem_pool_debugfs_max_size); -} - -/** - * set_simplified_mem_pool_max_size - Set the same maximum size for all the - * memory pools of small (4KiB/16KiB/64KiB) pages. - * @dev: The device with sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file - * @count: The number of bytes written to the sysfs file - * - * This function is called to set the same maximum size for all the memory - * pools of small pages. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t set_simplified_mem_pool_max_size(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - unsigned long new_size; - size_t gid; - int err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kstrtoul(buf, 0, &new_size); - if (err) - return -EINVAL; - - for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) - kbase_mem_pool_debugfs_set_max_size(kbdev->mem_pools.small, gid, (size_t)new_size); - - return (ssize_t)count; -} - -static DEVICE_ATTR(max_size, 0600, show_simplified_mem_pool_max_size, - set_simplified_mem_pool_max_size); - -/** - * show_simplified_lp_mem_pool_max_size - Show the maximum size for the memory - * pool 0 of large (2MiB) pages. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the total current pool size. - * - * This function is called to get the maximum size for the memory pool 0 of - * large (2MiB) pages. It is assumed that the maximum size value is same for - * all the pools. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t show_simplified_lp_mem_pool_max_size(struct device *dev, - struct device_attribute *attr, char *const buf) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, kbdev->mem_pools.large, 1, - kbase_mem_pool_debugfs_max_size); -} - -/** - * set_simplified_lp_mem_pool_max_size - Set the same maximum size for all the - * memory pools of large (2MiB) pages. - * @dev: The device with sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file - * @count: The number of bytes written to the sysfs file - * - * This function is called to set the same maximum size for all the memory - * pools of large (2MiB) pages. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t set_simplified_lp_mem_pool_max_size(struct device *dev, - struct device_attribute *attr, const char *buf, - size_t count) -{ - struct kbase_device *const kbdev = to_kbase_device(dev); - unsigned long new_size; - size_t gid; - int err; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - err = kstrtoul(buf, 0, &new_size); - if (err) - return -EINVAL; - - for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) - kbase_mem_pool_debugfs_set_max_size(kbdev->mem_pools.large, gid, (size_t)new_size); - - return (ssize_t)count; -} - -static DEVICE_ATTR(lp_max_size, 0600, show_simplified_lp_mem_pool_max_size, - set_simplified_lp_mem_pool_max_size); - -/** - * show_simplified_ctx_default_max_size - Show the default maximum size for the - * memory pool 0 of small (4KiB/16KiB/64KiB) pages. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the pool size. - * - * This function is called to get the default ctx maximum size for the memory - * pool 0 of small pages. It is assumed that maximum size value is same - * for all the pools. The maximum size for the pool of large (2MiB) pages will - * be same as max size of the pool of small pages in terms of bytes. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t show_simplified_ctx_default_max_size(struct device *dev, - struct device_attribute *attr, char *const buf) -{ - struct kbase_device *kbdev = to_kbase_device(dev); - size_t max_size; - - CSTD_UNUSED(attr); - - if (!kbdev) - return -ENODEV; - - max_size = kbase_mem_pool_config_debugfs_max_size(kbdev->mem_pool_defaults.small, 0); - - return scnprintf(buf, PAGE_SIZE, "%zu\n", max_size); -} - -/** - * set_simplified_ctx_default_max_size - Set the same default maximum size for - * all the pools created for new - * contexts. This covers the pool of - * large pages as well and its max size - * will be same as max size of the pool - * of small pages in terms of bytes. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. - * @count: The number of bytes written to the sysfs file. - * - * This function is called to set the same maximum size for all pools created - * for new contexts. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t set_simplified_ctx_default_max_size(struct device *dev, - struct device_attribute *attr, const char *buf, - size_t count) -{ - struct kbase_device *kbdev; - unsigned long new_size; - int err; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - err = kstrtoul(buf, 0, &new_size); - if (err) - return -EINVAL; - - kbase_mem_pool_group_config_set_max_size(&kbdev->mem_pool_defaults, (size_t)new_size); - - return (ssize_t)count; -} - -static DEVICE_ATTR(ctx_default_max_size, 0600, show_simplified_ctx_default_max_size, - set_simplified_ctx_default_max_size); - -#if !MALI_USE_CSF -/** - * js_ctx_scheduling_mode_show - Show callback for js_ctx_scheduling_mode sysfs - * entry. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The output buffer to receive the context scheduling mode information. - * - * This function is called to get the context scheduling mode being used by JS. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t js_ctx_scheduling_mode_show(struct device *dev, struct device_attribute *attr, - char *const buf) -{ - struct kbase_device *kbdev; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - return scnprintf(buf, PAGE_SIZE, "%u\n", kbdev->js_ctx_scheduling_mode); -} - -/** - * js_ctx_scheduling_mode_store - Set callback for js_ctx_scheduling_mode sysfs - * entry. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. - * @count: The number of bytes written to the sysfs file. - * - * This function is called when the js_ctx_scheduling_mode sysfs file is written - * to. It checks the data written, and if valid updates the ctx scheduling mode - * being by JS. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t js_ctx_scheduling_mode_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct kbase_context *kctx; - u32 new_js_ctx_scheduling_mode; - struct kbase_device *kbdev; - unsigned long flags; - int ret; - - CSTD_UNUSED(attr); - - kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - - ret = kstrtouint(buf, 0, &new_js_ctx_scheduling_mode); - if (ret || new_js_ctx_scheduling_mode >= KBASE_JS_PRIORITY_MODE_COUNT) { - dev_err(kbdev->dev, "Couldn't process js_ctx_scheduling_mode" - " write operation.\n" - "Use format \n"); - return -EINVAL; - } - - if (new_js_ctx_scheduling_mode == kbdev->js_ctx_scheduling_mode) - return (ssize_t)count; - - mutex_lock(&kbdev->kctx_list_lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* Update the context priority mode */ - kbdev->js_ctx_scheduling_mode = new_js_ctx_scheduling_mode; - - /* Adjust priority of all the contexts as per the new mode */ - list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) - kbase_js_update_ctx_priority(kctx); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->kctx_list_lock); - - dev_dbg(kbdev->dev, "JS ctx scheduling mode: %u\n", new_js_ctx_scheduling_mode); - - return (ssize_t)count; -} - -static DEVICE_ATTR_RW(js_ctx_scheduling_mode); - -/* Number of entries in serialize_jobs_settings[] */ -#define NR_SERIALIZE_JOBS_SETTINGS 5 -/* Maximum string length in serialize_jobs_settings[].name */ -#define MAX_SERIALIZE_JOBS_NAME_LEN 16 - -static struct { - char *name; - u8 setting; -} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { - { "none", 0 }, - { "intra-slot", KBASE_SERIALIZE_INTRA_SLOT }, - { "inter-slot", KBASE_SERIALIZE_INTER_SLOT }, - { "full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT }, - { "full-reset", - KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | KBASE_SERIALIZE_RESET } -}; - -/** - * update_serialize_jobs_setting - Update the serialization setting for the - * submission of GPU jobs. - * - * @kbdev: An instance of the GPU platform device, allocated from the probe - * method of the driver. - * @buf: Buffer containing the value written to the sysfs/debugfs file. - * @count: The number of bytes to write to the sysfs/debugfs file. - * - * This function is called when the serialize_jobs sysfs/debugfs file is - * written to. It matches the requested setting against the available settings - * and if a matching setting is found updates kbdev->serialize_jobs. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t update_serialize_jobs_setting(struct kbase_device *kbdev, const char *buf, - size_t count) -{ - int i; - bool valid = false; - - for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { - if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { - kbdev->serialize_jobs = serialize_jobs_settings[i].setting; - valid = true; - break; - } - } - - if (!valid) { - dev_err(kbdev->dev, "serialize_jobs: invalid setting"); - return -EINVAL; - } - - return (ssize_t)count; -} - -#if IS_ENABLED(CONFIG_DEBUG_FS) -/** - * kbasep_serialize_jobs_seq_debugfs_show - Show callback for the serialize_jobs - * debugfs file - * @sfile: seq_file pointer - * @data: Private callback data - * - * This function is called to get the contents of the serialize_jobs debugfs - * file. This is a list of the available settings with the currently active one - * surrounded by square brackets. - * - * Return: 0 on success, or an error code on error - */ -static int kbasep_serialize_jobs_seq_debugfs_show(struct seq_file *sfile, void *data) -{ - struct kbase_device *kbdev = sfile->private; - int i; - - CSTD_UNUSED(data); - - for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { - if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) - seq_printf(sfile, "[%s] ", serialize_jobs_settings[i].name); - else - seq_printf(sfile, "%s ", serialize_jobs_settings[i].name); - } - - seq_puts(sfile, "\n"); - - return 0; -} - -/** - * kbasep_serialize_jobs_debugfs_write - Store callback for the serialize_jobs - * debugfs file. - * @file: File pointer - * @ubuf: User buffer containing data to store - * @count: Number of bytes in user buffer - * @ppos: File position - * - * This function is called when the serialize_jobs debugfs file is written to. - * It matches the requested setting against the available settings and if a - * matching setting is found updates kbdev->serialize_jobs. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, const char __user *ubuf, - size_t count, loff_t *ppos) -{ - struct seq_file *s = file->private_data; - struct kbase_device *kbdev = s->private; - char buf[MAX_SERIALIZE_JOBS_NAME_LEN]; - - CSTD_UNUSED(ppos); - - count = min_t(size_t, sizeof(buf) - 1, count); - if (copy_from_user(buf, ubuf, count)) - return -EFAULT; - - buf[count] = 0; - - return update_serialize_jobs_setting(kbdev, buf, count); -} - -/** - * kbasep_serialize_jobs_debugfs_open - Open callback for the serialize_jobs - * debugfs file - * @in: inode pointer - * @file: file pointer - * - * Return: Zero on success, error code on failure - */ -static int kbasep_serialize_jobs_debugfs_open(struct inode *in, struct file *file) -{ - return single_open(file, kbasep_serialize_jobs_seq_debugfs_show, in->i_private); -} - -static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { - .owner = THIS_MODULE, - .open = kbasep_serialize_jobs_debugfs_open, - .read = seq_read, - .write = kbasep_serialize_jobs_debugfs_write, - .llseek = seq_lseek, - .release = single_release, -}; - -#endif /* CONFIG_DEBUG_FS */ - -/** - * show_serialize_jobs_sysfs - Show callback for serialize_jobs sysfs file. - * - * @dev: The device this sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The output buffer for the sysfs file contents - * - * This function is called to get the contents of the serialize_jobs sysfs - * file. This is a list of the available settings with the currently active - * one surrounded by square brackets. - * - * Return: The number of bytes output to @buf. - */ -static ssize_t show_serialize_jobs_sysfs(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct kbase_device *kbdev = to_kbase_device(dev); - ssize_t ret = 0; - int i; - - CSTD_UNUSED(attr); - - for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { - if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "[%s]", - serialize_jobs_settings[i].name); - else - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "%s ", - serialize_jobs_settings[i].name); - } - - if (ret < (ssize_t)(PAGE_SIZE - 1)) { - ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "\n"); - } else { - buf[PAGE_SIZE - 2] = '\n'; - buf[PAGE_SIZE - 1] = '\0'; - ret = PAGE_SIZE - 1; - } - - return ret; -} - -/** - * store_serialize_jobs_sysfs - Store callback for serialize_jobs sysfs file. - * - * @dev: The device this sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file - * @count: The number of bytes to write to the sysfs file - * - * This function is called when the serialize_jobs sysfs file is written to. - * It matches the requested setting against the available settings and if a - * matching setting is found updates kbdev->serialize_jobs. - * - * Return: @count if the function succeeded. An error code on failure. - */ -static ssize_t store_serialize_jobs_sysfs(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - CSTD_UNUSED(attr); - return update_serialize_jobs_setting(to_kbase_device(dev), buf, count); -} - -static DEVICE_ATTR(serialize_jobs, 0600, show_serialize_jobs_sysfs, store_serialize_jobs_sysfs); -#endif /* !MALI_USE_CSF */ - static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) { struct kbase_device *kbdev = @@ -4340,11 +2919,7 @@ static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) bool do_disable; -#if MALI_USE_CSF backend_lock = &kbdev->csf.scheduler.interrupt_lock; -#else - backend_lock = &kbdev->hwaccess_lock; -#endif spin_lock_irqsave(backend_lock, flags); do_disable = !kbdev->protected_mode_hwcnt_desired && !kbdev->protected_mode_hwcnt_disabled; @@ -4364,9 +2939,6 @@ static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) * the state machine. */ kbdev->protected_mode_hwcnt_disabled = true; -#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -#endif /* !MALI_USE_CSF */ } else { /* Protected mode state was updated while we were doing the * disable, so we need to undo the disable we just performed. @@ -4565,25 +3137,6 @@ int kbase_device_backend_init(struct kbase_device *kbdev) err = kbase_arbiter_pm_early_init(kbdev); if (err == 0) { -#if !MALI_USE_CSF - u32 product_model; - - /* - * Attempt to obtain and parse gpu_id in the event an external AW module - * is used for messaging. We should have access to GPU at this point. - */ - if (kbdev->gpu_props.gpu_id.arch_major == 0) - kbase_gpuprops_parse_gpu_id(&kbdev->gpu_props.gpu_id, - kbase_reg_get_gpu_id(kbdev)); - - product_model = kbdev->gpu_props.gpu_id.product_model; - if (product_model != GPU_ID_PRODUCT_TGOX && product_model != GPU_ID_PRODUCT_TNOX && - product_model != GPU_ID_PRODUCT_TBAX) { - kbase_arbiter_pm_early_term(kbdev); - dev_dbg(kbdev->dev, "GPU platform not suitable for arbitration"); - return -EPERM; - } -#endif /* !MALI_USE_CSF */ dev_info(kbdev->dev, "Arbitration interface enabled"); } #endif /* defined(CONFIG_OF) */ @@ -4787,9 +3340,7 @@ MAKE_QUIRK_ACCESSORS(sc); MAKE_QUIRK_ACCESSORS(tiler); MAKE_QUIRK_ACCESSORS(mmu); MAKE_QUIRK_ACCESSORS(gpu); -#if MALI_USE_CSF MAKE_QUIRK_ACCESSORS(ne); -#endif /** * kbase_device_debugfs_reset_write() - Reset the GPU @@ -4859,36 +3410,52 @@ static const struct file_operations fops_protected_debug_mode = { .llseek = default_llseek, }; -static int kbase_device_debugfs_mem_pool_max_size_show(struct seq_file *sfile, void *data) +static int kbase_device_debugfs_heap_reclaim_offslot_show(struct seq_file *sfile, void *data) { + struct kbase_csf_heap_reclaim_offslot *const setting = sfile->private; + CSTD_UNUSED(data); - return kbase_debugfs_helper_seq_read(sfile, MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_config_debugfs_max_size); + + seq_printf(sfile, "%u %u\n", setting->timeout_ms, setting->pages); + + return 0; } -static ssize_t kbase_device_debugfs_mem_pool_max_size_write(struct file *file, - const char __user *ubuf, size_t count, - loff_t *ppos) +static ssize_t kbase_device_debugfs_heap_reclaim_offslot_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) { - ssize_t err = 0; + const struct seq_file *const sfile = (struct seq_file *)file->private_data; + struct kbase_csf_heap_reclaim_offslot *const setting = sfile->private; + u32 read_len, timeout_ms, pages; + char kbuf[32]; CSTD_UNUSED(ppos); - err = kbase_debugfs_helper_seq_write(file, ubuf, count, MEMORY_GROUP_MANAGER_NR_GROUPS, - kbase_mem_pool_config_debugfs_set_max_size); - return err ? err : (ssize_t)count; + read_len = count < (sizeof(kbuf) - 1) ? count : (sizeof(kbuf) - 1); + if (copy_from_user(kbuf, ubuf, read_len)) + return -EFAULT; + kbuf[read_len] = '\0'; + + if (sscanf(kbuf, "%u %u", &timeout_ms, &pages) != 2) + return -EINVAL; + + setting->timeout_ms = timeout_ms; + setting->pages = pages; + + return count; } -static int kbase_device_debugfs_mem_pool_max_size_open(struct inode *in, struct file *file) +static int kbase_device_debugfs_heap_reclaim_offslot_open(struct inode *in, struct file *file) { - return single_open(file, kbase_device_debugfs_mem_pool_max_size_show, in->i_private); + return single_open(file, kbase_device_debugfs_heap_reclaim_offslot_show, in->i_private); } -static const struct file_operations kbase_device_debugfs_mem_pool_max_size_fops = { +static const struct file_operations kbase_device_debugfs_heap_reclaim_offslot_fops = { .owner = THIS_MODULE, - .open = kbase_device_debugfs_mem_pool_max_size_open, + .open = kbase_device_debugfs_heap_reclaim_offslot_open, .read = seq_read, - .write = kbase_device_debugfs_mem_pool_max_size_write, + .write = kbase_device_debugfs_heap_reclaim_offslot_write, .llseek = seq_lseek, .release = single_release, }; @@ -4916,20 +3483,6 @@ static struct dentry *debugfs_ctx_defaults_init(struct kbase_device *const kbdev debugfs_create_bool("infinite_cache", mode, debugfs_ctx_defaults_directory, &kbdev->infinite_cache_active_default); - dentry = debugfs_create_file("mem_pool_max_size", mode, debugfs_ctx_defaults_directory, - &kbdev->mem_pool_defaults.small, - &kbase_device_debugfs_mem_pool_max_size_fops); - if (IS_ERR_OR_NULL(dentry)) { - dev_err(kbdev->dev, "Unable to create mem_pool_max_size debugfs entry\n"); - return dentry; - } - - dentry = debugfs_create_file("lp_mem_pool_max_size", mode, debugfs_ctx_defaults_directory, - &kbdev->mem_pool_defaults.large, - &kbase_device_debugfs_mem_pool_max_size_fops); - if (IS_ERR_OR_NULL(dentry)) - dev_err(kbdev->dev, "Unable to create lp_mem_pool_max_size debugfs entry\n"); - return dentry; } @@ -4965,11 +3518,7 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) kbasep_regs_history_debugfs_init(kbdev); -#if MALI_USE_CSF kbase_debug_csf_fault_debugfs_init(kbdev); -#else /* MALI_USE_CSF */ - kbase_debug_job_fault_debugfs_init(kbdev); -#endif /* !MALI_USE_CSF */ kbasep_gpu_memory_debugfs_init(kbdev); kbase_as_fault_debugfs_init(kbdev); @@ -4977,6 +3526,7 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) kbase_instr_backend_debugfs_init(kbdev); #endif kbase_pbha_debugfs_init(kbdev); + kbase_gpu_timestamp_offset_debugfs_init(kbdev); /* fops_* variables created by invocations of macro * MAKE_QUIRK_ACCESSORS() above. @@ -5009,7 +3559,6 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) return dentry; } -#if MALI_USE_CSF if (kbase_csf_dev_has_ne(kbdev)) { dentry = debugfs_create_file("quirks_ne", 0644, kbdev->mali_debugfs_directory, kbdev, &fops_ne_quirks); @@ -5021,7 +3570,6 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) if (kbase_csf_ne_control_debugfs_init(kbdev)) return NULL; } -#endif dentry = debugfs_ctx_defaults_init(kbdev); if (IS_ERR_OR_NULL(dentry)) @@ -5045,6 +3593,20 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) return dentry; } + { + struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; + struct kbase_csf_sched_heap_reclaim_mgr *const mgr = &scheduler->reclaim_mgr; + + dentry = debugfs_create_file("heap_reclaim_offslot", 0644, + kbdev->mali_debugfs_directory, &mgr->offslot_setting, + &kbase_device_debugfs_heap_reclaim_offslot_fops); + if (IS_ERR_OR_NULL(dentry)) { + dev_err(kbdev->dev, + "Unable to create heap_reclaim_offslot debugfs entry\n"); + return dentry; + } + } + kbase_ktrace_debugfs_init(kbdev); #ifdef CONFIG_MALI_VALHALL_DEVFREQ @@ -5054,15 +3616,6 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) #endif /* CONFIG_DEVFREQ_THERMAL */ #endif /* CONFIG_MALI_VALHALL_DEVFREQ */ -#if !MALI_USE_CSF - dentry = debugfs_create_file("serialize_jobs", 0644, kbdev->mali_debugfs_directory, kbdev, - &kbasep_serialize_jobs_debugfs_fops); - if (IS_ERR_OR_NULL(dentry)) { - dev_err(kbdev->dev, "Unable to create serialize_jobs debugfs entry\n"); - return dentry; - } - kbase_timeline_io_debugfs_init(kbdev); -#endif kbase_dvfs_status_debugfs_init(kbdev); @@ -5140,7 +3693,7 @@ static bool kbase_device_supports_coherency_mode(struct kbase_device *kbdev, u32 * on CSF GPUs. */ if (coherency_mode == COHERENCY_ACE) { - if (IS_ENABLED(MALI_USE_CSF) && !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI)) { + if (!IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI)) { dev_err(kbdev->dev, "ACE coherency not supported on CSF, wrong DT configuration"); return false; @@ -5208,8 +3761,6 @@ early_exit: } -#if MALI_USE_CSF - bool kbasep_adjust_prioritized_process(struct kbase_device *kbdev, bool add, uint32_t tgid) { struct kbase_context *kctx; @@ -5723,33 +4274,16 @@ static ssize_t mcu_shader_pwroff_timeout_ns_store(struct device *dev, struct dev static DEVICE_ATTR_RW(mcu_shader_pwroff_timeout_ns); -#endif /* MALI_USE_CSF */ - -static struct attribute *kbase_scheduling_attrs[] = { -#if !MALI_USE_CSF - &dev_attr_serialize_jobs.attr, -#endif /* !MALI_USE_CSF */ - NULL -}; +static struct attribute *kbase_scheduling_attrs[] = { NULL }; static struct attribute *kbase_attrs[] = { #ifdef CONFIG_MALI_VALHALL_DEBUG &dev_attr_debug_command.attr, -#if !MALI_USE_CSF - &dev_attr_js_softstop_always.attr, -#endif /* !MALI_USE_CSF */ #endif -#if !MALI_USE_CSF - &dev_attr_js_timeouts.attr, - &dev_attr_soft_job_timeout.attr, -#endif /* !MALI_USE_CSF */ &dev_attr_gpuinfo.attr, &dev_attr_dvfs_period.attr, &dev_attr_pm_poweroff.attr, &dev_attr_reset_timeout.attr, -#if !MALI_USE_CSF - &dev_attr_js_scheduling_period.attr, -#else &dev_attr_csg_scheduling_period.attr, &dev_attr_add_prioritized_process.attr, &dev_attr_remove_prioritized_process.attr, @@ -5758,35 +4292,17 @@ static struct attribute *kbase_attrs[] = { &dev_attr_idle_hysteresis_time_ns.attr, &dev_attr_mcu_shader_pwroff_timeout.attr, &dev_attr_mcu_shader_pwroff_timeout_ns.attr, -#endif /* !MALI_USE_CSF */ &dev_attr_power_policy.attr, &dev_attr_core_mask.attr, - &dev_attr_mem_pool_size.attr, - &dev_attr_mem_pool_max_size.attr, - &dev_attr_lp_mem_pool_size.attr, - &dev_attr_lp_mem_pool_max_size.attr, -#if !MALI_USE_CSF - &dev_attr_js_ctx_scheduling_mode.attr, -#endif /* !MALI_USE_CSF */ NULL }; -static struct attribute *kbase_mempool_attrs[] = { &dev_attr_max_size.attr, - &dev_attr_lp_max_size.attr, - &dev_attr_ctx_default_max_size.attr, NULL }; - #define SYSFS_SCHEDULING_GROUP "scheduling" static const struct attribute_group kbase_scheduling_attr_group = { .name = SYSFS_SCHEDULING_GROUP, .attrs = kbase_scheduling_attrs, }; -#define SYSFS_MEMPOOL_GROUP "mempool" -static const struct attribute_group kbase_mempool_attr_group = { - .name = SYSFS_MEMPOOL_GROUP, - .attrs = kbase_mempool_attrs, -}; - static const struct attribute_group kbase_attr_group = { .attrs = kbase_attrs, }; @@ -5812,19 +4328,11 @@ int kbase_sysfs_init(struct kbase_device *kbdev) return err; } - err = sysfs_create_group(&kbdev->dev->kobj, &kbase_mempool_attr_group); - if (err) { - dev_err(kbdev->dev, "Creation of %s sysfs group failed", SYSFS_MEMPOOL_GROUP); - sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); - sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); - } - return err; } void kbase_sysfs_term(struct kbase_device *kbdev) { - sysfs_remove_group(&kbdev->dev->kobj, &kbase_mempool_attr_group); sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); put_device(kbdev->dev); @@ -6013,10 +4521,6 @@ static int kbase_device_resume(struct device *dev) kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_RESUME); #endif -#if !MALI_USE_CSF - kbase_enable_quick_reset(kbdev); -#endif - return 0; } @@ -6031,23 +4535,23 @@ static int kbase_device_resume(struct device *dev) * * Return: A standard Linux error code */ -#ifdef KBASE_PM_RUNTIME static int kbase_device_runtime_suspend(struct device *dev) { struct kbase_device *kbdev = to_kbase_device(dev); int ret = 0; + if (!IS_ENABLED(CONFIG_PM)) + return 0; + if (!kbdev) return -ENODEV; dev_dbg(dev, "Callback %s\n", __func__); KBASE_KTRACE_ADD(kbdev, PM_RUNTIME_SUSPEND_CALLBACK, NULL, 0); -#if MALI_USE_CSF ret = kbase_pm_handle_runtime_suspend(kbdev); if (ret) return ret; -#endif #ifdef CONFIG_MALI_VALHALL_DVFS kbase_pm_metrics_stop(kbdev); @@ -6063,9 +4567,8 @@ static int kbase_device_runtime_suspend(struct device *dev) kbdev->is_runtime_resumed = false; dev_dbg(dev, "runtime suspend\n"); } - return ret; + return 0; } -#endif /* KBASE_PM_RUNTIME */ /** * kbase_device_runtime_resume - Runtime resume callback from the OS. @@ -6077,12 +4580,14 @@ static int kbase_device_runtime_suspend(struct device *dev) * Return: A standard Linux error code */ -#ifdef KBASE_PM_RUNTIME static int kbase_device_runtime_resume(struct device *dev) { int ret = 0; struct kbase_device *kbdev = to_kbase_device(dev); + if (!IS_ENABLED(CONFIG_PM)) + return 0; + if (!kbdev) return -ENODEV; @@ -6105,9 +4610,7 @@ static int kbase_device_runtime_resume(struct device *dev) return ret; } -#endif /* KBASE_PM_RUNTIME */ -#ifdef KBASE_PM_RUNTIME /** * kbase_device_runtime_idle - Runtime idle callback from the OS. * @dev: The device to suspend @@ -6122,6 +4625,9 @@ static int kbase_device_runtime_idle(struct device *dev) { struct kbase_device *kbdev = to_kbase_device(dev); + if (!IS_ENABLED(CONFIG_PM)) + return 0; + if (!kbdev) return -ENODEV; @@ -6136,18 +4642,15 @@ static int kbase_device_runtime_idle(struct device *dev) pm_runtime_mark_last_busy(kbdev->dev); return 0; } -#endif /* KBASE_PM_RUNTIME */ /* The power management operations for the platform driver. */ static const struct dev_pm_ops kbase_pm_ops = { SYSTEM_SLEEP_PM_OPS(kbase_device_suspend, kbase_device_resume) -#ifdef KBASE_PM_RUNTIME .runtime_suspend = kbase_device_runtime_suspend, .runtime_resume = kbase_device_runtime_resume, .runtime_idle = kbase_device_runtime_idle, -#endif /* KBASE_PM_RUNTIME */ }; #if IS_ENABLED(CONFIG_OF) diff --git a/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.c b/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.c index e64a5715b126..f9b3069e7128 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.c +++ b/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.c @@ -24,11 +24,7 @@ #include #include "mali_kbase_ctx_sched.h" #include "tl/mali_kbase_tracepoints.h" -#if MALI_USE_CSF #include "mali_kbase_reset_gpu.h" -#else -#include -#endif /* Helper for ktrace */ #if KBASE_KTRACE_ENABLE @@ -152,19 +148,12 @@ void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx) struct kbase_device *const kbdev = kctx->kbdev; lockdep_assert_held(&kbdev->hwaccess_lock); -#if MALI_USE_CSF /* We expect the context to be active when this function is called, * except for the case where a page fault is reported for it during * the GPU reset sequence, in which case we can expect the refcount * to be 0. */ WARN_ON(!atomic_read(&kctx->refcount) && !kbase_reset_gpu_is_active(kbdev)); -#else - /* We expect the context to be active (and thus refcount should be non-zero) - * when this function is called - */ - WARN_ON(!atomic_read(&kctx->refcount)); -#endif if (likely((kctx->as_nr >= 0) && (kctx->as_nr < BASE_MAX_NR_AS))) WARN_ON(kbdev->as_to_kctx[kctx->as_nr] != kctx); else @@ -188,9 +177,6 @@ void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) kbdev->as_to_kctx[kctx->as_nr] = NULL; kctx->as_nr = KBASEP_AS_NR_INVALID; kbase_ctx_flag_clear(kctx, KCTX_AS_DISABLED_ON_FAULT); -#if !MALI_USE_CSF - kbase_backend_slot_kctx_purge_locked(kbdev, kctx); -#endif } } } @@ -206,6 +192,19 @@ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) mutex_lock(&kbdev->mmu_hw_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + kbase_ctx_sched_remove_ctx_nolock(kctx); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->mmu_hw_mutex); +} + +void kbase_ctx_sched_remove_ctx_nolock(struct kbase_context *kctx) +{ + struct kbase_device *const kbdev = kctx->kbdev; + + lockdep_assert_held(&kbdev->mmu_hw_mutex); + lockdep_assert_held(&kbdev->hwaccess_lock); + WARN_ON(atomic_read(&kctx->refcount) != 0); if ((kctx->as_nr >= 0) && (kctx->as_nr < BASE_MAX_NR_AS)) { @@ -215,9 +214,6 @@ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) kbdev->as_to_kctx[kctx->as_nr] = NULL; kctx->as_nr = KBASEP_AS_NR_INVALID; } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); } void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) @@ -234,12 +230,10 @@ void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { struct kbase_context *kctx; -#if MALI_USE_CSF if ((i == MCU_AS_NR) && kbdev->csf.firmware_inited) { kbase_mmu_update(kbdev, &kbdev->csf.mcu_mmu, MCU_AS_NR); continue; } -#endif kctx = kbdev->as_to_kctx[i]; if (kctx) { if (atomic_read(&kctx->refcount)) { @@ -378,7 +372,6 @@ void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx) spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); } -#if MALI_USE_CSF bool kbase_ctx_sched_inc_refcount_if_as_valid(struct kbase_context *kctx) { struct kbase_device *kbdev; @@ -412,4 +405,3 @@ bool kbase_ctx_sched_inc_refcount_if_as_valid(struct kbase_context *kctx) return added_ref; } -#endif diff --git a/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.h b/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.h index fd1b82471d26..bdc9e8fa4784 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.h +++ b/drivers/gpu/arm/valhall/mali_kbase_ctx_sched.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -130,6 +130,20 @@ void kbase_ctx_sched_release_ctx(struct kbase_context *kctx); */ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx); +/** + * kbase_ctx_sched_remove_ctx_nolock - Unassign previously assigned address space + * @kctx: The context to be removed + * + * The following lock must be held by the caller: + * kbase_device::mmu_hw_mutex + * kbase_device::hwaccess_lock + * + * This function should be called when a context is being destroyed. The + * context must no longer have any reference. If it has been assigned an + * address space before then the AS will be unprogrammed. + */ +void kbase_ctx_sched_remove_ctx_nolock(struct kbase_context *kctx); + /** * kbase_ctx_sched_restore_all_as - Reprogram all address spaces * @kbdev: The device for which address spaces to be reprogrammed @@ -228,7 +242,6 @@ bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx); */ void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx); -#if MALI_USE_CSF /** * kbase_ctx_sched_inc_refcount_if_as_valid - Refcount the context if it has GPU * address space slot assigned to it. @@ -244,6 +257,5 @@ void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx); * was not assigned). */ bool kbase_ctx_sched_inc_refcount_if_as_valid(struct kbase_context *kctx); -#endif #endif /* _KBASE_CTX_SCHED_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.c b/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.c deleted file mode 100644 index 679dd75187db..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.c +++ /dev/null @@ -1,544 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include - -#if IS_ENABLED(CONFIG_DEBUG_FS) - -static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev) -{ - struct list_head *event_list = &kbdev->job_fault_event_list; - unsigned long flags; - bool ret; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - ret = !list_empty(event_list); - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - - return ret; -} - -static void kbase_ctx_remove_pending_event(struct kbase_context *kctx) -{ - struct list_head *event_list = &kctx->kbdev->job_fault_event_list; - struct base_job_fault_event *event; - unsigned long flags; - - spin_lock_irqsave(&kctx->kbdev->job_fault_event_lock, flags); - list_for_each_entry(event, event_list, head) { - if (event->katom->kctx == kctx) { - list_del(&event->head); - WARN_ON_ONCE(&event->job_fault_work != kctx->job_fault_work); - wake_up(&kctx->kbdev->job_fault_resume_wq); - /* job_fault_event_list can only have a single atom for - * each context. - */ - break; - } - } - spin_unlock_irqrestore(&kctx->kbdev->job_fault_event_lock, flags); - if (kctx->job_fault_work) - flush_work(kctx->job_fault_work); -} - -static bool kbase_ctx_has_no_event_pending(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - struct list_head *event_list = &kctx->kbdev->job_fault_event_list; - struct base_job_fault_event *event; - unsigned long flags; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - if (list_empty(event_list)) { - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - return true; - } - list_for_each_entry(event, event_list, head) { - if (event->katom->kctx == kctx) { - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - return false; - } - } - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - return true; -} - -static int wait_for_job_fault(struct kbase_device *kbdev) -{ -#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE - int ret = wait_event_interruptible_timeout(kbdev->job_fault_wq, - kbase_is_job_fault_event_pending(kbdev), - msecs_to_jiffies(2000)); - if (ret == 0) - return -EAGAIN; - else if (ret > 0) - return 0; - else - return ret; -#else - return wait_event_interruptible(kbdev->job_fault_wq, - kbase_is_job_fault_event_pending(kbdev)); -#endif -} - -/* wait until the fault happen and copy the event */ -static int kbase_job_fault_event_wait(struct kbase_device *kbdev, - struct base_job_fault_event *event) -{ - struct list_head *event_list = &kbdev->job_fault_event_list; - struct base_job_fault_event *event_in; - unsigned long flags; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - while (list_empty(event_list)) { - int err; - - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - - err = wait_for_job_fault(kbdev); - if (err) - return err; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - } - - event_in = list_entry(event_list->next, struct base_job_fault_event, head); - event->event_code = event_in->event_code; - event->katom = event_in->katom; - - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - - return 0; -} - -/* remove the event from the queue */ -static struct base_job_fault_event *kbase_job_fault_event_dequeue(struct kbase_device *kbdev, - struct list_head *event_list) -{ - struct base_job_fault_event *event; - - CSTD_UNUSED(kbdev); - - event = list_entry(event_list->next, struct base_job_fault_event, head); - list_del(event_list->next); - - return event; -} - -/* Remove all the following atoms after the failed atom in the same context - * Call the postponed bottom half of job done. - * Then, this context could be rescheduled. - */ -static void kbase_job_fault_resume_event_cleanup(struct kbase_context *kctx) -{ - struct list_head *event_list = &kctx->job_fault_resume_event_list; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - while (!list_empty(event_list)) { - struct base_job_fault_event *event; - - event = kbase_job_fault_event_dequeue(kctx->kbdev, - &kctx->job_fault_resume_event_list); - WARN_ON(work_pending(&event->katom->work)); - INIT_WORK(&event->katom->work, kbase_jd_done_worker); - queue_work(kctx->jctx.job_done_wq, &event->katom->work); - } -} - -static void kbase_job_fault_resume_worker(struct work_struct *data) -{ - struct base_job_fault_event *event = - container_of(data, struct base_job_fault_event, job_fault_work); - struct kbase_device *kbdev; - struct kbase_context *kctx; - struct kbase_jd_atom *katom; - unsigned long flags; - - katom = event->katom; - kctx = katom->kctx; - kbdev = kctx->kbdev; - - dev_info(kbdev->dev, "Job dumping wait\n"); - - /* When it was waked up, it need to check if queue is empty or the - * failed atom belongs to different context. If yes, wake up. Both - * of them mean the failed job has been dumped. Please note, it - * should never happen that the job_fault_event_list has the two - * atoms belong to the same context. - */ - wait_event(kbdev->job_fault_resume_wq, kbase_ctx_has_no_event_pending(kctx)); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - atomic_set(&kctx->job_fault_count, 0); - WARN_ON(work_pending(&katom->work)); - INIT_WORK(&katom->work, kbase_jd_done_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); - - /* In case the following atoms were scheduled during failed job dump - * the job_done_worker was held. We need to rerun it after the dump - * was finished - */ - kbase_job_fault_resume_event_cleanup(kctx); - dev_info(kbdev->dev, "Job dumping finish, resume scheduler\n"); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -static struct base_job_fault_event *kbase_job_fault_event_queue(struct list_head *event_list, - struct kbase_jd_atom *atom, - u32 completion_code) -{ - struct base_job_fault_event *event; - - event = &atom->fault_event; - - event->katom = atom; - event->event_code = completion_code; - - list_add_tail(&event->head, event_list); - - return event; -} - -static void kbase_job_fault_event_post(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - u32 completion_code) -{ - struct base_job_fault_event *event; - unsigned long flags; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - event = kbase_job_fault_event_queue(&kbdev->job_fault_event_list, katom, completion_code); - INIT_WORK(&event->job_fault_work, kbase_job_fault_resume_worker); - katom->kctx->job_fault_work = &event->job_fault_work; - wake_up_interruptible(&kbdev->job_fault_wq); - queue_work(kbdev->job_fault_resume_workq, &event->job_fault_work); - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - - dev_info(katom->kctx->kbdev->dev, "Job fault happen, start dump: %d_%d", katom->kctx->tgid, - katom->kctx->id); -} - -/* - * This function will process the job fault - * Get the register copy - * Send the failed job dump event - * Create a Wait queue to wait until the job dump finish - */ - -bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, u32 completion_code) -{ - struct kbase_context *kctx = katom->kctx; - - /* Check if dumping is in the process - * only one atom of each context can be dumped at the same time - * If the atom belongs to different context, it can be dumped - */ - if (atomic_read(&kctx->job_fault_count) > 0) { - kbase_job_fault_event_queue(&kctx->job_fault_resume_event_list, katom, - completion_code); - dev_info(kctx->kbdev->dev, "queue:%d\n", kbase_jd_atom_id(kctx, katom)); - return true; - } - - if (kbase_ctx_flag(kctx, KCTX_DYING)) - return false; - - if (atomic_read(&kctx->kbdev->job_fault_debug) > 0) { - if (completion_code != BASE_JD_EVENT_DONE) { - if (kbase_job_fault_get_reg_snapshot(kctx) == false) { - dev_warn(kctx->kbdev->dev, "get reg dump failed\n"); - return false; - } - - kbase_job_fault_event_post(kctx->kbdev, katom, completion_code); - atomic_inc(&kctx->job_fault_count); - dev_info(kctx->kbdev->dev, "post:%d\n", kbase_jd_atom_id(kctx, katom)); - return true; - } - } - return false; -} - -static int debug_job_fault_show(struct seq_file *m, void *v) -{ - struct kbase_device *kbdev = m->private; - struct base_job_fault_event *event = (struct base_job_fault_event *)v; - struct kbase_context *kctx = event->katom->kctx; - int i; - - dev_info(kbdev->dev, "debug job fault seq show:%d_%d, %d", kctx->tgid, kctx->id, - event->reg_offset); - - if (kctx->reg_dump == NULL) { - dev_warn(kbdev->dev, "reg dump is NULL"); - return -1; - } - - if (kctx->reg_dump[event->reg_offset] == REGISTER_DUMP_TERMINATION_FLAG) { - /* Return the error here to stop the read. And the - * following next() will not be called. The stop can - * get the real event resource and release it - */ - return -1; - } - - if (event->reg_offset == 0) - seq_printf(m, "%d_%d\n", kctx->tgid, kctx->id); - - for (i = 0; i < 50; i++) { - if (kctx->reg_dump[event->reg_offset] == REGISTER_DUMP_TERMINATION_FLAG) { - break; - } - seq_printf(m, "%08x: %08x\n", kctx->reg_dump[event->reg_offset], - kctx->reg_dump[1 + event->reg_offset]); - event->reg_offset += 2; - } - - return 0; -} -static void *debug_job_fault_next(struct seq_file *m, void *v, loff_t *pos) -{ - struct kbase_device *kbdev = m->private; - struct base_job_fault_event *event = (struct base_job_fault_event *)v; - - dev_info(kbdev->dev, "debug job fault seq next:%d, %d", event->reg_offset, (int)*pos); - - return event; -} - -static void *debug_job_fault_start(struct seq_file *m, loff_t *pos) -{ - struct kbase_device *kbdev = m->private; - struct base_job_fault_event *event; - - dev_info(kbdev->dev, "fault job seq start:%d", (int)*pos); - - /* The condition is trick here. It needs make sure the - * fault hasn't happened and the dumping hasn't been started, - * or the dumping has finished - */ - if (*pos == 0) { - event = kmalloc(sizeof(*event), GFP_KERNEL); - if (!event) - return NULL; - event->reg_offset = 0; - if (kbase_job_fault_event_wait(kbdev, event)) { - kfree(event); - return NULL; - } - - /* The cache flush workaround is called in bottom half of - * job done but we delayed it. Now we should clean cache - * earlier. Then the GPU memory dump should be correct. - */ - kbase_backend_cache_clean(kbdev, event->katom); - } else - return NULL; - - return event; -} - -static void debug_job_fault_stop(struct seq_file *m, void *v) -{ - struct kbase_device *kbdev = m->private; - - /* here we wake up the kbase_jd_done_worker after stop, it needs - * get the memory dump before the register dump in debug daemon, - * otherwise, the memory dump may be incorrect. - */ - - if (v != NULL) { - kfree(v); - dev_info(kbdev->dev, "debug job fault seq stop stage 1"); - - } else { - unsigned long flags; - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - if (!list_empty(&kbdev->job_fault_event_list)) { - kbase_job_fault_event_dequeue(kbdev, &kbdev->job_fault_event_list); - wake_up(&kbdev->job_fault_resume_wq); - } - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - dev_info(kbdev->dev, "debug job fault seq stop stage 2"); - } -} - -static const struct seq_operations ops = { - .start = debug_job_fault_start, - .next = debug_job_fault_next, - .stop = debug_job_fault_stop, - .show = debug_job_fault_show, -}; - -static int debug_job_fault_open(struct inode *in, struct file *file) -{ - struct kbase_device *kbdev = in->i_private; - - if (atomic_cmpxchg(&kbdev->job_fault_debug, 0, 1) == 1) { - dev_warn(kbdev->dev, "debug job fault is busy, only a single client is allowed"); - return -EBUSY; - } - - seq_open(file, &ops); - - ((struct seq_file *)file->private_data)->private = kbdev; - dev_info(kbdev->dev, "debug job fault seq open"); - - return 0; -} - -static int debug_job_fault_release(struct inode *in, struct file *file) -{ - struct kbase_device *kbdev = in->i_private; - struct list_head *event_list = &kbdev->job_fault_event_list; - unsigned long flags; - - seq_release(in, file); - - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - - /* Disable job fault dumping. This will let kbase run jobs as normal, - * without blocking waiting for a job_fault client to read failed jobs. - * - * After this a new client may open the file, and may re-enable job - * fault dumping, but the job_fault_event_lock we hold here will block - * that from interfering until after we've completed the cleanup. - */ - atomic_dec(&kbdev->job_fault_debug); - - /* Clean the unprocessed job fault. After that, all the suspended - * contexts could be rescheduled. Remove all the failed atoms that - * belong to different contexts Resume all the contexts that were - * suspend due to failed job. - */ - while (!list_empty(event_list)) { - kbase_job_fault_event_dequeue(kbdev, event_list); - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - wake_up(&kbdev->job_fault_resume_wq); - spin_lock_irqsave(&kbdev->job_fault_event_lock, flags); - } - - spin_unlock_irqrestore(&kbdev->job_fault_event_lock, flags); - - dev_info(kbdev->dev, "debug job fault seq close"); - - return 0; -} - -static const struct file_operations kbasep_debug_job_fault_fops = { - .owner = THIS_MODULE, - .open = debug_job_fault_open, - .read = seq_read, - .llseek = seq_lseek, - .release = debug_job_fault_release, -}; - -/* - * Initialize debugfs entry for job fault dump - */ -void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev) -{ - debugfs_create_file("job_fault", 0400, kbdev->mali_debugfs_directory, kbdev, - &kbasep_debug_job_fault_fops); -} - -int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) -{ - INIT_LIST_HEAD(&kbdev->job_fault_event_list); - - init_waitqueue_head(&(kbdev->job_fault_wq)); - init_waitqueue_head(&(kbdev->job_fault_resume_wq)); - spin_lock_init(&kbdev->job_fault_event_lock); - - kbdev->job_fault_resume_workq = - alloc_workqueue("kbase_job_fault_resume_work_queue", WQ_MEM_RECLAIM, 1); - if (!kbdev->job_fault_resume_workq) - return -ENOMEM; - - return 0; -} - -/* - * Release the relevant resource per device - */ -void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) -{ - destroy_workqueue(kbdev->job_fault_resume_workq); -} - -/* - * Initialize the relevant data structure per context - */ -int kbase_debug_job_fault_context_init(struct kbase_context *kctx) -{ - /* We need allocate double size register range - * Because this memory will keep the register address and value - */ - kctx->reg_dump = vmalloc(0x4000 * 2); - if (kctx->reg_dump != NULL) { - if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { - vfree(kctx->reg_dump); - kctx->reg_dump = NULL; - } - INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); - } - - return 0; -} - -/* - * release the relevant resource per context - */ -void kbase_debug_job_fault_context_term(struct kbase_context *kctx) -{ - vfree(kctx->reg_dump); -} - -void kbase_debug_job_fault_kctx_unblock(struct kbase_context *kctx) -{ - WARN_ON(!kbase_ctx_flag(kctx, KCTX_DYING)); - - /* Return early if the job fault part of the kbase_device is not - * initialized yet. An error can happen during the device probe after - * the privileged Kbase context was created for the HW counter dumping - * but before the job fault part is initialized. - */ - if (!kctx->kbdev->job_fault_resume_workq) - return; - - kbase_ctx_remove_pending_event(kctx); -} - -#else /* CONFIG_DEBUG_FS */ - -int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev) -{ - return 0; -} - -void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) -{ -} - -#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.h b/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.h deleted file mode 100644 index ee1228155621..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_debug_job_fault.h +++ /dev/null @@ -1,125 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_DEBUG_JOB_FAULT_H -#define _KBASE_DEBUG_JOB_FAULT_H - -#include - -struct kbase_context; -struct kbase_device; - -#define REGISTER_DUMP_TERMINATION_FLAG 0xFFFFFFFF - -/** - * kbase_debug_job_fault_dev_init - Create the fault event wait queue - * per device and initialize the required lists. - * @kbdev: Device pointer - * - * This function must be called only when a kbase device is initialized. - * - * Return: Zero on success or a negative error code. - */ -int kbase_debug_job_fault_dev_init(struct kbase_device *kbdev); - -/** - * kbase_debug_job_fault_debugfs_init - Initialize job fault debug sysfs - * @kbdev: Device pointer - */ -void kbase_debug_job_fault_debugfs_init(struct kbase_device *kbdev); - -/** - * kbase_debug_job_fault_dev_term - Clean up resources created in - * kbase_debug_job_fault_dev_init. - * @kbdev: Device pointer - */ -void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); - -/** - * kbase_debug_job_fault_context_init - Initialize the relevant data structure per context - * @kctx: KBase context pointer - * - * This function must be called only when a kbase context is instantiated. - * - * Return: 0 on success - */ -int kbase_debug_job_fault_context_init(struct kbase_context *kctx); - -/** - * kbase_debug_job_fault_context_term - Release the relevant - * resource per context - * @kctx: KBase context pointer - */ -void kbase_debug_job_fault_context_term(struct kbase_context *kctx); - -/** - * kbase_debug_job_fault_kctx_unblock - Unblock the atoms blocked on job fault - * dumping on context termination. - * - * @kctx: KBase context pointer - * - * This function is called during context termination to unblock the atom for - * which the job fault occurred and also the atoms following it. This is needed - * otherwise the wait for zero jobs could timeout (leading to an assertion - * failure, kernel panic in debug builds) in the pathological case where - * although the thread/daemon capturing the job fault events is running, - * but for some reasons has stopped consuming the events. - */ -void kbase_debug_job_fault_kctx_unblock(struct kbase_context *kctx); - -/** - * kbase_debug_job_fault_process - Process the failed job. - * - * @katom: The failed atom pointer - * @completion_code: the job status - * - * It will send a event and wake up the job fault waiting queue - * Then create a work queue to wait for job dump finish - * This function should be called in the interrupt handler and before - * jd_done that make sure the jd_done_worker will be delayed until the - * job dump finish - * - * Return: true if dump is going on - */ -bool kbase_debug_job_fault_process(struct kbase_jd_atom *katom, u32 completion_code); - -/** - * kbase_debug_job_fault_reg_snapshot_init - Set the interested registers - * address during the job fault process, the relevant registers will - * be saved when a job fault happen - * @kctx: KBase context pointer - * @reg_range: Maximum register address space - * - * Return: true if initializing successfully - */ -bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, int reg_range); - -/** - * kbase_job_fault_get_reg_snapshot - Read the interested registers for - * failed job dump - * - * @kctx: KBase context pointer - * - * Return: true if getting registers successfully - */ -bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx); - -#endif /*_KBASE_DEBUG_JOB_FAULT_H*/ diff --git a/drivers/gpu/arm/valhall/mali_kbase_debug_mem_zones.c b/drivers/gpu/arm/valhall/mali_kbase_debug_mem_zones.c index 01267b599d89..3964b5505bdd 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_debug_mem_zones.c +++ b/drivers/gpu/arm/valhall/mali_kbase_debug_mem_zones.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -63,7 +63,6 @@ static int debug_mem_zones_show(struct seq_file *sfile, void *data) reg_zone->va_size_pages); } } -#if MALI_USE_CSF reg_zone = &kctx->kbdev->csf.mcu_shared_zone; if (reg_zone && reg_zone->base_pfn) { @@ -71,7 +70,6 @@ static int debug_mem_zones_show(struct seq_file *sfile, void *data) kbase_reg_zone_get_name(MCU_SHARED_ZONE), MCU_SHARED_ZONE, reg_zone->base_pfn, reg_zone->va_size_pages); } -#endif kbase_gpu_vm_unlock(kctx); return 0; diff --git a/drivers/gpu/arm/valhall/mali_kbase_defs.h b/drivers/gpu/arm/valhall/mali_kbase_defs.h index 978e76c3fdcf..ccfeacae678b 100755 --- a/drivers/gpu/arm/valhall/mali_kbase_defs.h +++ b/drivers/gpu/arm/valhall/mali_kbase_defs.h @@ -39,12 +39,7 @@ #include #include -#if MALI_USE_CSF #include -#else -#include -#include -#endif #include "debug/mali_kbase_debug_ktrace_defs.h" @@ -126,7 +121,6 @@ */ #define KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS (200 * NSEC_PER_USEC) -#if MALI_USE_CSF /* The buffer count of CSF hwcnt backend ring buffer, which is used when CSF * hwcnt backend allocate the ring buffer to communicate with CSF firmware for * HWC dump samples. @@ -134,7 +128,6 @@ * CSF hwcnt backend creation will be failed. */ #define KBASE_HWCNT_BACKEND_CSF_RING_BUFFER_COUNT (128) -#endif /* Maximum number of clock/regulator pairs that may be referenced by * the device node. @@ -150,7 +143,6 @@ struct kbase_context; struct kbase_device; struct kbase_as; struct kbase_mmu_setup; -struct kbase_kinstr_jm; struct kbase_io; #if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) @@ -358,11 +350,7 @@ struct kbase_mmu_table { #endif }; -#if MALI_USE_CSF #include "csf/mali_kbase_csf_defs.h" -#else -#include "jm/mali_kbase_jm_defs.h" -#endif #include "mali_kbase_hwaccess_time.h" @@ -459,6 +447,12 @@ struct kbase_clk_rate_trace_manager { * has already been called from the power_on callback, then * the call to it from runtime_gpu_active callback can be * skipped. + * @runtime_suspend_result: Stores the return value of + * kbase_pm_handle_runtime_suspend() to be passed back + * to the kernel. This is consumed in + * kbase_device_runtime_suspend() once the runtime + * suspend event has been handled by the CSF scheduler + * kthread. * @gpu_lost: Flag indicating gpu lost * This structure contains data for the power management framework. * There is one instance of this structure per device in the system. @@ -466,8 +460,8 @@ struct kbase_clk_rate_trace_manager { * @resume_wait: Wait queue to wait for the System suspend/resume of GPU device. * @debug_core_mask: Bit masks identifying the available shader cores that are * specified via sysfs. One mask per job slot. - * @debug_core_mask_all: Bit masks identifying the available shader cores that - * are specified via sysfs. + * @sysfs_gov_core_mask: Bit masks identifying the available shader cores that are + * specified via sysfs when writing to GOV_CORE_MASK. * @callback_power_runtime_init: Callback for initializing the runtime power * management. Return 0 on success, else error code * @callback_power_runtime_term: Callback for terminating the runtime power @@ -485,20 +479,14 @@ struct kbase_pm_device_data { int active_count; bool suspending; bool resuming; -#if MALI_USE_CSF bool runtime_active; -#endif + int runtime_suspend_result; atomic_t gpu_lost; wait_queue_head_t zero_active_count_wait; wait_queue_head_t resume_wait; -#if MALI_USE_CSF u64 debug_core_mask; -#else - /* One mask per job slot. */ - u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS]; - u64 debug_core_mask_all; -#endif /* MALI_USE_CSF */ + u64 sysfs_gov_core_mask; int (*callback_power_runtime_init)(struct kbase_device *kbdev); void (*callback_power_runtime_term)(struct kbase_device *kbdev); @@ -530,13 +518,12 @@ struct kbase_pm_device_data { * @isolation_in_progress_cnt: Number of pages in pool undergoing page isolation. * This is used to avoid race condition between pool termination * and page isolation for page migration. - * @next_pool: Pointer to next pool where pages can be allocated when this - * pool is empty. Pages will spill over to the next pool when - * this pool is full. Can be NULL if there is no next pool. * @dying: true if the pool is being terminated, and any ongoing * operations should be abandoned - * @dont_reclaim: true if the shrinker is forbidden from reclaiming memory from - * this pool, eg during a grow operation + * @pool_supports_reclaim: Whether this pool supports page reclaiming. + * @reclaim_allowed: true if the shrinker is currently allowed to reclaim from this + * pool. Otherwise, false: the shrinker is forbidden from reclaiming + * memory from it - eg during a grow operation. */ struct kbase_mem_pool { struct kbase_device *kbdev; @@ -549,10 +536,9 @@ struct kbase_mem_pool { DEFINE_KBASE_SHRINKER reclaim; atomic_t isolation_in_progress_cnt; - struct kbase_mem_pool *next_pool; - bool dying; - bool dont_reclaim; + bool pool_supports_reclaim; + bool reclaim_allowed; }; /** @@ -599,6 +585,20 @@ struct kbase_mem_pool_group_config { struct kbase_mem_pool_config large[MEMORY_GROUP_MANAGER_NR_GROUPS]; }; +/** + * struct kbase_fw_mem_pool_group - a set of physical memory pools for fw load. + * + * @small: Object containing the state for pool of small size + * physical pages. + * @large: Object containing the state for pool of large size + * physical pages. + * + */ +struct kbase_fw_mem_pool_group { + struct kbase_mem_pool small; + struct kbase_mem_pool large; +}; + /** * struct kbase_devfreq_opp - Lookup table for converting between nominal OPP * frequency, real frequencies and core mask @@ -819,12 +819,11 @@ struct kbase_mem_migrate { * framework. * @fw_load_lock: Mutex to protect firmware loading in @ref kbase_open. * @csf: CSF object for the GPU device. - * @js_data: Per device object encapsulating the current context of - * Job Scheduler, which is global to the device and is not - * tied to any particular struct kbase_context running on - * the device - * @mem_pools: Global pools of free physical memory pages which can - * be used by all the contexts. + * @pgd_mem_pool: Global pool of free physical memory pages dedicated to + * allocation of pgd pages that are not associated with + * contexts. + * @fw_mem_pools: Global pools of free physical memory pages which can + * be used by firmware load subroutine. * @memdev: keeps track of the in use physical pages allocated by * the Driver. * @mmu_mode: Pointer to the object containing methods for programming @@ -857,17 +856,6 @@ struct kbase_mem_migrate { * @nr_user_address_spaces: Number of address spaces available to user contexts * @hwcnt_backend_csf_if_fw: Firmware interface to access CSF GPU performance * counters. - * @hwcnt: Structure used for instrumentation and HW counters - * dumping - * @hwcnt.lock: The lock should be used when accessing any of the - * following members - * @hwcnt.kctx: kbase context - * @hwcnt.addr: HW counter address - * @hwcnt.addr_bytes: HW counter size in bytes - * @hwcnt.backend: Kbase instrumentation backend - * @hwcnt_gpu_jm_backend: Job manager GPU backend interface, used as superclass reference - * pointer by hwcnt_gpu_iface, which wraps this implementation in - * order to extend it with periodic dumping functionality. * @hwcnt_gpu_iface: Backend interface for GPU hardware counter access. * @hwcnt_watchdog_timer: Watchdog interface, used by the GPU backend hwcnt_gpu_iface to * perform periodic dumps in order to prevent hardware counter value @@ -955,26 +943,12 @@ struct kbase_mem_migrate { * @previous_frequency: Previous frequency of GPU clock used for * KBASE_HW_ISSUE_GPU2017_1336 workaround, This clock is * restored when L2 is powered on. - * @job_fault_debug: Flag to control the dumping of debug data for job faults, - * set when the 'job_fault' debugfs file is opened. * @mali_debugfs_directory: Root directory for the debugfs files created by the driver * @debugfs_ctx_directory: Directory inside the @mali_debugfs_directory containing * a sub-directory for every context. * @debugfs_instr_directory: Instrumentation debugfs directory * @debugfs_as_read_bitmap: bitmap of address spaces for which the bus or page fault * has occurred. - * @job_fault_wq: Waitqueue to block the job fault dumping daemon till the - * occurrence of a job fault. - * @job_fault_resume_wq: Waitqueue on which every context with a faulty job wait - * for the job fault dumping to complete before they can - * do bottom half of job done for the atoms which followed - * the faulty atom. - * @job_fault_resume_workq: workqueue to process the work items queued for the faulty - * atoms, whereby the work item function waits for the dumping - * to get completed. - * @job_fault_event_list: List of atoms, each belonging to a different context, which - * generated a job fault. - * @job_fault_event_lock: Lock to protect concurrent accesses to @job_fault_event_list * @regs_dump_debugfs_data: Contains the offset of register to be read through debugfs * file "read_register". * @regs_dump_debugfs_data.reg_offset: Contains the offset of register to be @@ -995,8 +969,6 @@ struct kbase_mem_migrate { * power on for GPU is started. * @infinite_cache_active_default: Set to enable using infinite cache for all the * allocations of a new context. - * @mem_pool_defaults: Default configuration for the group of memory pools - * created for a new context. * @current_gpu_coherency_mode: coherency mode in use, which can be different * from @system_coherency, when using protected mode. * @system_coherency: coherency mode as retrieved from the device tree. @@ -1029,10 +1001,6 @@ struct kbase_mem_migrate { * the updates made to Job dispatcher + scheduler states. * @mmu_hw_mutex: Protects access to MMU operations and address space * related state. - * @serialize_jobs: Currently used mode for serialization of jobs, both - * intra & inter slots serialization is supported. - * @js_ctx_scheduling_mode: Context scheduling mode currently being used by - * Job Scheduler * @l2_size_override: Used to set L2 cache size via device tree blob * @l2_hash_override: Used to set L2 cache hash via device tree blob * @l2_hash_values_override: true if @l2_hash_values is valid. @@ -1086,9 +1054,7 @@ struct kbase_device { u32 hw_quirks_tiler; u32 hw_quirks_mmu; u32 hw_quirks_gpu; -#if MALI_USE_CSF u32 hw_quirks_ne; -#endif struct list_head entry; struct device *dev; @@ -1128,7 +1094,8 @@ struct kbase_device { #endif /* !IS_ENABLED(CONFIG_MALI_VALHALL_REAL_HW) */ struct kbase_pm_device_data pm; - struct kbase_mem_pool_group mem_pools; + struct kbase_mem_pool pgd_mem_pool; + struct kbase_fw_mem_pool_group fw_mem_pools; struct kbasep_mem_device memdev; struct kbase_mmu_mode const *mmu_mode; @@ -1166,21 +1133,7 @@ struct kbase_device { */ u32 mma_wa_id; -#if MALI_USE_CSF struct kbase_hwcnt_backend_csf_if hwcnt_backend_csf_if_fw; -#else - struct kbase_hwcnt { - spinlock_t lock; - - struct kbase_context *kctx; - u64 addr; - u64 addr_bytes; - - struct kbase_instr_backend backend; - } hwcnt; - - struct kbase_hwcnt_backend_interface hwcnt_gpu_jm_backend; -#endif struct kbase_hwcnt_backend_interface hwcnt_gpu_iface; struct kbase_hwcnt_watchdog_interface hwcnt_watchdog_timer; @@ -1253,10 +1206,6 @@ struct kbase_device { #endif /* CONFIG_MALI_VALHALL_DEVFREQ */ unsigned long previous_frequency; -#if !MALI_USE_CSF - atomic_t job_fault_debug; -#endif /* !MALI_USE_CSF */ - struct dentry *mali_debugfs_directory; struct dentry *debugfs_ctx_directory; struct dentry *debugfs_instr_directory; @@ -1265,14 +1214,6 @@ struct kbase_device { u64 debugfs_as_read_bitmap; #endif /* CONFIG_MALI_VALHALL_DEBUG */ -#if !MALI_USE_CSF - wait_queue_head_t job_fault_wq; - wait_queue_head_t job_fault_resume_wq; - struct workqueue_struct *job_fault_resume_workq; - struct list_head job_fault_event_list; - spinlock_t job_fault_event_lock; -#endif /* !MALI_USE_CSF */ - #if !MALI_CUSTOMER_RELEASE struct { u32 reg_offset; @@ -1294,8 +1235,6 @@ struct kbase_device { bool infinite_cache_active_default; - struct kbase_mem_pool_group_config mem_pool_defaults; - u32 current_gpu_coherency_mode; u32 system_coherency; @@ -1335,19 +1274,8 @@ struct kbase_device { u32 sysc_alloc[GPU_SYSC_ALLOC_COUNT]; struct mutex fw_load_lock; -#if MALI_USE_CSF /* CSF object for the GPU device. */ struct kbase_csf_device csf; -#else - struct kbasep_js_device_data js_data; - - /* See KBASE_JS_*_PRIORITY_MODE for details. */ - u32 js_ctx_scheduling_mode; - - /* See KBASE_SERIALIZE_* for details */ - u8 serialize_jobs; - -#endif /* MALI_USE_CSF */ struct rb_root process_root; struct rb_root dma_buf_root; @@ -1370,22 +1298,9 @@ struct kbase_device { struct notifier_block oom_notifier_block; -#if !MALI_USE_CSF - spinlock_t quick_reset_lock; - bool quick_reset_enabled; - /* - * 进入 quck_reset_mode 后 (quick_reset_enabled 为 true), - * 对已经进入 KBASE_JD_ATOM_STATE_HW_COMPLETED 状态的 atom 的计数. - * - * 若 num_of_atoms_hw_completed 达到一定值, 将退出 quck_reset_mode. - * 见 kbase_js_complete_atom() 对 num_of_atoms_hw_completed 的引用. - */ - u32 num_of_atoms_hw_completed; -#endif - struct kbase_mem_migrate mem_migrate; -#if MALI_USE_CSF && IS_ENABLED(CONFIG_SYNC_FILE) +#if IS_ENABLED(CONFIG_SYNC_FILE) atomic_t live_fence_metadata; #endif struct kmem_cache *va_region_slab; @@ -1397,9 +1312,7 @@ struct kbase_device { */ struct kbase_gpu_metrics gpu_metrics; #endif -#if MALI_USE_CSF atomic_t fence_signal_timeout_enabled; -#endif struct notifier_block pcm_prioritized_process_nb; @@ -1652,9 +1565,6 @@ struct kbase_sub_alloc { * events to Userspace. * @event_mutex: Lock to protect the concurrent access to @event_list & * @event_mutex. - * @event_closed: Flag set through POST_TERM ioctl, indicates that Driver - * should stop posting events and also inform event handling - * thread that context termination is in progress. * @event_workq: Workqueue for processing work items corresponding to atoms * that do not return an event to userspace. * @event_count: Count of the posted events to be consumed by Userspace. @@ -1710,8 +1620,6 @@ struct kbase_sub_alloc { * an application becomes foreground or goes to the * background. * @csf: kbase csf context - * @jctx: object encapsulating all the Job dispatcher related state, - * including the array of atoms. * @used_pages: Keeps a track of the number of small physical pages in use * for the context. * @nonmapped_pages: Updated in the same way as @used_pages, except for the case @@ -1719,6 +1627,8 @@ struct kbase_sub_alloc { * is reset to 0. * @permanent_mapped_pages: Usage count of permanently mapped memory * @mem_pools: Context-specific pools of free physical memory pages. + * Note that context-specific PGDs are not allocated from this. + * @pgd_mem_pool: Context-specific pool dedicated to allocation of PGDs. * @reclaim: Shrinker object registered with the kernel containing * the pointer to callback function which is invoked under * low memory conditions. In the callback function Driver @@ -1787,23 +1697,6 @@ struct kbase_sub_alloc { * output of "mem_view" debugfs file. * @job_fault_work: Tracking the latest fault dump work item for assisting the * operation of the job-fault-dump debug process. - * @jsctx_queue: Per slot & priority arrays of object containing the root - * of RB-tree holding currently runnable atoms on the job slot - * and the head item of the linked list of atoms blocked on - * cross-slot dependencies. - * @slot_tracking: Tracking and control of this context's use of all job - * slots - * @atoms_pulled_all_slots: Total number of atoms currently pulled from the - * context, across all slots. - * @slots_pullable: Bitmask of slots, indicating the slots for which the - * context has pullable atoms in the runnable tree. - * @completed_jobs: List containing completed atoms for which base_jd_event is - * to be posted. - * @work_count: Number of work items, corresponding to atoms, currently - * pending on job_done workqueue of @jctx. - * @soft_job_timeout: Timer object used for failing/cancelling the waiting - * soft-jobs which have been blocked for more than the - * timeout value used for the soft-jobs * @jit_alloc: Array of 256 pointers to GPU memory regions, used for * just-in-time memory allocations. * @jit_max_allocations: Maximum allowed number of in-flight @@ -1858,30 +1751,22 @@ struct kbase_sub_alloc { * @ext_res_meta_head: A list of sticky external resources which were requested to * be mapped on GPU side, through a softjob atom of type * EXT_RES_MAP or STICKY_RESOURCE_MAP ioctl. - * @age_count: Counter incremented on every call to jd_submit_atom, - * atom is assigned the snapshot of this counter, which - * is used to determine the atom's age when it is added to - * the runnable RB-tree. * @trim_level: Level of JIT allocation trimming to perform on free (0-100%) * @kprcs: Reference to @struct kbase_process that the current * kbase_context belongs to. * @kprcs_link: List link for the list of kbase context maintained * under kbase_process. - * @priority: Indicates the context priority. Used along with @atoms_count - * for context scheduling, protected by hwaccess_lock. - * @atoms_count: Number of GPU atoms currently in use, per priority * @create_flags: Flags used in context creation. - * @kinstr_jm: Kernel job manager instrumentation context handle * @tl_kctx_list_node: List item into the device timeline's list of * contexts, for timeline summarization. * @limited_core_mask: The mask that is applied to the affinity in case of atoms * marked with BASE_JD_REQ_LIMITED_CORE_MASK. - * @platform_data: Pointer to platform specific per-context data. * @task: Pointer to the task structure of the main thread of the process * that created the Kbase context. It would be set only for the * contexts created by the Userspace and not for the contexts * created internally by the Kbase. * @comm: Record the process name + * @offslot_ts: System time in ns when all CSGs belonged to this kctx become offslot. * * A kernel base context is an entity among which the GPU is scheduled. * Each context has its own GPU address space. @@ -1900,9 +1785,6 @@ struct kbase_context { struct list_head event_list; struct list_head event_coalesce_list; struct mutex event_mutex; -#if !MALI_USE_CSF - atomic_t event_closed; -#endif struct workqueue_struct *event_workq; atomic_t event_count; int event_coalesce_count; @@ -1915,29 +1797,11 @@ struct kbase_context { struct list_head mem_partials; struct mutex reg_lock; -#if MALI_USE_CSF atomic64_t num_fixable_allocs; atomic64_t num_fixed_allocs; -#endif struct kbase_reg_zone reg_zone[CONTEXT_ZONE_MAX]; -#if MALI_USE_CSF struct kbase_csf_context csf; -#else - struct kbase_jd_context jctx; - struct jsctx_queue jsctx_queue[KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS]; - struct kbase_jsctx_slot_tracking slot_tracking[BASE_JM_MAX_NR_SLOTS]; - atomic_t atoms_pulled_all_slots; - - struct list_head completed_jobs; - atomic_t work_count; - struct timer_list soft_job_timeout; - - int priority; - s16 atoms_count[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; - u32 slots_pullable; - u32 age_count; -#endif /* MALI_USE_CSF */ DECLARE_BITMAP(cookies, BITS_PER_LONG); struct kbase_va_region *pending_regions[BITS_PER_LONG]; @@ -1951,6 +1815,7 @@ struct kbase_context { atomic_t permanent_mapped_pages; struct kbase_mem_pool_group mem_pools; + struct kbase_mem_pool pgd_mem_pool; DEFINE_KBASE_SHRINKER reclaim; @@ -1966,11 +1831,9 @@ struct kbase_context { struct mm_struct *process_mm; u64 gpu_va_end; -#if MALI_USE_CSF u32 running_total_tiler_heap_nr_chunks; u64 running_total_tiler_heap_memory; u64 peak_total_tiler_heap_memory; -#endif bool jit_va; #if IS_ENABLED(CONFIG_DEBUG_FS) @@ -2011,17 +1874,10 @@ struct kbase_context { base_context_create_flags create_flags; -#if !MALI_USE_CSF - struct kbase_kinstr_jm *kinstr_jm; -#endif struct list_head tl_kctx_list_node; u64 limited_core_mask; -#if !MALI_USE_CSF - void *platform_data; -#endif - struct task_struct *task; #if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) @@ -2033,6 +1889,8 @@ struct kbase_context { #endif char comm[TASK_COMM_LEN]; + + u64 offslot_ts; }; /** diff --git a/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.c b/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.c deleted file mode 100644 index 10bbb64f7def..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.c +++ /dev/null @@ -1,404 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Implementation of the dummy job execution workaround for the GPU hang issue. - */ - -#include -#include -#include -#include - -#include -#include - -#define DUMMY_JOB_WA_BINARY_NAME "valhall-1691526.wa" - -struct wa_header { - u16 signature; - u16 version; - u32 info_offset; -} __packed; - -struct wa_v2_info { - u64 jc; - u32 js; - u32 blob_offset; - u64 flags; -} __packed; - -struct wa_blob { - u64 base; - u32 size; - u32 map_flags; - u32 payload_offset; - u32 blob_offset; -} __packed; - -static bool within_range(const u8 *base, const u8 *end, off_t off, size_t sz) -{ - return !((size_t)(end - base - off) < sz); -} - -static u32 wait_any(struct kbase_device *kbdev, off_t offset, u32 bits) -{ - int loop; - const int timeout = 100; - u32 val; - - for (loop = 0; loop < timeout; loop++) { - val = kbase_reg_read32(kbdev, offset); - if (val & bits) - break; - udelay(10); - } - - if (loop == timeout) { - dev_err(kbdev->dev, - "Timeout reading register 0x%lx, bits 0x%lx, last read was 0x%lx\n", - (unsigned long)offset, (unsigned long)bits, (unsigned long)val); - } - - return (val & bits); -} - -static inline int run_job(struct kbase_device *kbdev, int as, u32 slot, u64 cores, u64 jc) -{ - u32 done; - - /* setup job */ - kbase_reg_write64(kbdev, JOB_SLOT_OFFSET(slot, HEAD_NEXT), jc); - kbase_reg_write64(kbdev, JOB_SLOT_OFFSET(slot, AFFINITY_NEXT), cores); - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(slot, CONFIG_NEXT), - JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK | (unsigned int)as); - - /* go */ - kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(slot, COMMAND_NEXT), JS_COMMAND_START); - - /* wait for the slot to finish (done, error) */ - done = wait_any(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_RAWSTAT), - (1ul << (16 + slot)) | (1ul << slot)); - kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_CLEAR), done); - - if (done != (1ul << slot)) { - dev_err(kbdev->dev, "Failed to run WA job on slot %u cores 0x%llx: done 0x%lx\n", - slot, (unsigned long long)cores, (unsigned long)done); - dev_err(kbdev->dev, "JS_STATUS on failure: 0x%x\n", - kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(slot, STATUS))); - - return -EFAULT; - } else { - return 0; - } -} - -/* To be called after power up & MMU init, but before everything else */ -int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores) -{ - int as; - u32 slot; - u64 jc; - int failed = 0; - int runs = 0; - u32 old_gpu_mask; - u32 old_job_mask; - u64 val; - const u32 timeout_us = 10000; - - if (!kbdev) - return -EFAULT; - - if (!kbdev->dummy_job_wa.kctx) - return -EFAULT; - - as = kbdev->dummy_job_wa.kctx->as_nr; - slot = kbdev->dummy_job_wa.slot; - jc = kbdev->dummy_job_wa.jc; - - /* mask off all but MMU IRQs */ - old_gpu_mask = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)); - old_job_mask = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK)); - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), 0); - kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK), 0); - - /* power up requested cores */ - kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(SHADER_PWRON), cores); - - if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP) { - /* wait for power-ups */ - kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(SHADER_READY), val, - (val & cores) == cores, 10, timeout_us, false); - } - - if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE) { - size_t i; - - /* do for each requested core */ - for (i = 0; i < sizeof(cores) * 8; i++) { - u64 affinity; - - affinity = 1ull << i; - - if (!(cores & affinity)) - continue; - - if (run_job(kbdev, as, slot, affinity, jc)) - failed++; - runs++; - } - - } else { - if (run_job(kbdev, as, slot, cores, jc)) - failed++; - runs++; - } - - if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) { - /* power off shader cores (to reduce any dynamic leakage) */ - kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(SHADER_PWROFF), cores); - - /* wait for power off complete */ - kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(SHADER_READY), val, !(val & cores), - 10, timeout_us, false); - kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(SHADER_PWRTRANS), val, - !(val & cores), 10, timeout_us, false); - - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_CLEAR), U32_MAX); - } - - /* restore IRQ masks */ - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK), old_gpu_mask); - kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK), old_job_mask); - - if (failed) - dev_err(kbdev->dev, "WA complete with %d failures out of %d runs\n", failed, runs); - - return failed ? -EFAULT : 0; -} - -static ssize_t dummy_job_wa_info_show(struct device *const dev, struct device_attribute *const attr, - char *const buf) -{ - struct kbase_device *const kbdev = dev_get_drvdata(dev); - int err; - - CSTD_UNUSED(attr); - - if (!kbdev || !kbdev->dummy_job_wa.kctx) - return -ENODEV; - - err = scnprintf(buf, PAGE_SIZE, "slot %u flags %llx\n", kbdev->dummy_job_wa.slot, - kbdev->dummy_job_wa.flags); - - return err; -} - -static DEVICE_ATTR_RO(dummy_job_wa_info); - -static bool wa_blob_load_needed(struct kbase_device *kbdev) -{ - if (kbdev->gpu_props.impl_tech >= THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_FPGA) - return false; - - if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_3485)) - return true; - - return false; -} - -int kbase_dummy_job_wa_load(struct kbase_device *kbdev) -{ - const struct firmware *firmware; - static const char wa_name[] = DUMMY_JOB_WA_BINARY_NAME; - const u32 signature = 0x4157; - const u32 version = 2; - const u8 *fw_end; - const u8 *fw; - const struct wa_header *header; - const struct wa_v2_info *v2_info; - u32 blob_offset; - int err; - struct kbase_context *kctx; - - /* Calls to this function are inherently asynchronous, with respect to - * MMU operations. - */ - const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC; - - lockdep_assert_held(&kbdev->fw_load_lock); - - if (!wa_blob_load_needed(kbdev)) - return 0; - - /* load the wa */ - err = request_firmware(&firmware, wa_name, kbdev->dev); - - if (err) { - dev_err(kbdev->dev, - "WA blob missing. Please refer to the Arm Mali DDK Valhall Release Notes, " - "Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed"); - return -ENODEV; - } - - kctx = kbase_create_context(kbdev, true, BASE_CONTEXT_CREATE_FLAG_NONE, 0, NULL); - - if (!kctx) { - dev_err(kbdev->dev, "Failed to create WA context\n"); - goto no_ctx; - } - - fw = firmware->data; - fw_end = fw + firmware->size; - - dev_dbg(kbdev->dev, "Loaded firmware of size %zu bytes\n", firmware->size); - - if (!within_range(fw, fw_end, 0, sizeof(*header))) { - dev_err(kbdev->dev, "WA too small\n"); - goto bad_fw; - } - - header = (const struct wa_header *)(fw + 0); - - if (header->signature != signature) { - dev_err(kbdev->dev, "WA signature failure: 0x%lx\n", - (unsigned long)header->signature); - goto bad_fw; - } - - if (header->version != version) { - dev_err(kbdev->dev, "WA version 0x%lx not supported\n", - (unsigned long)header->version); - goto bad_fw; - } - - if (!within_range(fw, fw_end, header->info_offset, sizeof(*v2_info))) { - dev_err(kbdev->dev, "WA info offset out of bounds\n"); - goto bad_fw; - } - - v2_info = (const struct wa_v2_info *)(fw + header->info_offset); - - if (v2_info->flags & ~KBASE_DUMMY_JOB_WA_FLAGS) { - dev_err(kbdev->dev, "Unsupported WA flag(s): 0x%llx\n", - (unsigned long long)v2_info->flags); - goto bad_fw; - } - - kbdev->dummy_job_wa.slot = v2_info->js; - kbdev->dummy_job_wa.jc = v2_info->jc; - kbdev->dummy_job_wa.flags = v2_info->flags; - - blob_offset = v2_info->blob_offset; - - while (blob_offset) { - const struct wa_blob *blob; - size_t nr_pages; - base_mem_alloc_flags flags; - u64 gpu_va; - struct kbase_va_region *va_region; - - if (!within_range(fw, fw_end, blob_offset, sizeof(*blob))) { - dev_err(kbdev->dev, "Blob offset out-of-range: 0x%lx\n", - (unsigned long)blob_offset); - goto bad_fw; - } - - blob = (const struct wa_blob *)(fw + blob_offset); - if (!within_range(fw, fw_end, blob->payload_offset, blob->size)) { - dev_err(kbdev->dev, "Payload out-of-bounds\n"); - goto bad_fw; - } - - gpu_va = blob->base; - if (PAGE_ALIGN(gpu_va) != gpu_va) { - dev_err(kbdev->dev, "blob not page aligned\n"); - goto bad_fw; - } - nr_pages = PFN_UP(blob->size); - flags = blob->map_flags | BASEP_MEM_FLAG_MAP_FIXED; - - va_region = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, &gpu_va, - mmu_sync_info); - - if (!va_region) { - dev_err(kbdev->dev, "Failed to allocate for blob\n"); - } else { - struct kbase_vmap_struct vmap = { 0 }; - const u8 *payload; - void *dst; - - /* copy the payload, */ - payload = fw + blob->payload_offset; - - dst = kbase_vmap(kctx, va_region->start_pfn << PAGE_SHIFT, - nr_pages << PAGE_SHIFT, &vmap); - - if (dst) { - memcpy(dst, payload, blob->size); - kbase_vunmap(kctx, &vmap); - } else { - dev_err(kbdev->dev, "Failed to copy payload\n"); - } - } - blob_offset = blob->blob_offset; /* follow chain */ - } - - release_firmware(firmware); - - kbasep_js_schedule_privileged_ctx(kbdev, kctx); - - kbdev->dummy_job_wa.kctx = kctx; - - err = sysfs_create_file(&kbdev->dev->kobj, &dev_attr_dummy_job_wa_info.attr); - if (err) - dev_err(kbdev->dev, "SysFS file creation for dummy job wa failed\n"); - - return 0; - -bad_fw: - kbase_destroy_context(kctx); -no_ctx: - release_firmware(firmware); - return -EFAULT; -} - -void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) -{ - struct kbase_context *wa_kctx; - - /* return if the dummy job has not been loaded */ - if (kbdev->dummy_job_wa_loaded == false) - return; - - /* Can be safely called even if the file wasn't created on probe */ - sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_dummy_job_wa_info.attr); - - wa_kctx = READ_ONCE(kbdev->dummy_job_wa.kctx); - WRITE_ONCE(kbdev->dummy_job_wa.kctx, NULL); - /* make this write visible before we tear down the ctx */ - smp_mb(); - - if (wa_kctx) { - kbasep_js_release_privileged_ctx(kbdev, wa_kctx); - kbase_destroy_context(wa_kctx); - } -} diff --git a/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.h b/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.h deleted file mode 100644 index b46197aa6df3..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_dummy_job_wa.h +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_DUMMY_JOB_WORKAROUND_ -#define _KBASE_DUMMY_JOB_WORKAROUND_ - -#define KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE (1ull << 0) -#define KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP (1ull << 1) -#define KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER (1ull << 2) - -#define KBASE_DUMMY_JOB_WA_FLAGS \ - (KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE | KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP | \ - KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) - -#if MALI_USE_CSF - -static inline int kbase_dummy_job_wa_load(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - return 0; -} - -static inline void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} - -static inline int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores) -{ - CSTD_UNUSED(kbdev); - CSTD_UNUSED(cores); - return 0; -} - -static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); - return false; -} - -#else - -int kbase_dummy_job_wa_load(struct kbase_device *kbdev); -void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev); -int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores); - -static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev) -{ - return (kbdev->dummy_job_wa.kctx != NULL); -} - -#endif /* MALI_USE_CSF */ - -#endif /* _KBASE_DUMMY_JOB_WORKAROUND_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_dvfs_debugfs.c b/drivers/gpu/arm/valhall/mali_kbase_dvfs_debugfs.c index e912178b3324..f33a484a6796 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_dvfs_debugfs.c +++ b/drivers/gpu/arm/valhall/mali_kbase_dvfs_debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -38,16 +38,10 @@ static int kbasep_dvfs_utilization_debugfs_show(struct seq_file *file, void *dat struct kbase_device *kbdev = file->private; CSTD_UNUSED(data); -#if MALI_USE_CSF seq_printf(file, "busy_time: %u idle_time: %u protm_time: %u\n", kbdev->pm.backend.metrics.values.time_busy, kbdev->pm.backend.metrics.values.time_idle, kbdev->pm.backend.metrics.values.time_in_protm); -#else - seq_printf(file, "busy_time: %u idle_time: %u\n", - kbdev->pm.backend.metrics.values.time_busy, - kbdev->pm.backend.metrics.values.time_idle); -#endif return 0; } diff --git a/drivers/gpu/arm/valhall/mali_kbase_event.c b/drivers/gpu/arm/valhall/mali_kbase_event.c deleted file mode 100644 index 3c59b20913f2..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_event.c +++ /dev/null @@ -1,265 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include -#include - -static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, - struct kbase_jd_atom *katom) -{ - struct base_jd_udata data; - struct kbase_device *kbdev; - - lockdep_assert_held(&kctx->jctx.lock); - - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(katom != NULL); - KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); - - kbdev = kctx->kbdev; - data = katom->udata; - - KBASE_TLSTREAM_TL_NRET_ATOM_CTX(kbdev, katom, kctx); - KBASE_TLSTREAM_TL_DEL_ATOM(kbdev, katom); - - katom->status = KBASE_JD_ATOM_STATE_UNUSED; - dev_dbg(kbdev->dev, "Atom %pK status to unused\n", (void *)katom); - wake_up(&katom->completed); - - return data; -} - -int kbase_event_dequeue(struct kbase_context *kctx, struct base_jd_event_v2 *uevent) -{ - struct kbase_jd_atom *atom; - - KBASE_DEBUG_ASSERT(kctx); - - mutex_lock(&kctx->event_mutex); - - if (list_empty(&kctx->event_list)) { - if (!atomic_read(&kctx->event_closed)) { - mutex_unlock(&kctx->event_mutex); - return -1; - } - - /* generate the BASE_JD_EVENT_DRV_TERMINATED message on the fly */ - mutex_unlock(&kctx->event_mutex); - uevent->event_code = BASE_JD_EVENT_DRV_TERMINATED; - memset(&uevent->udata, 0, sizeof(uevent->udata)); - dev_dbg(kctx->kbdev->dev, - "event system closed, returning BASE_JD_EVENT_DRV_TERMINATED(0x%X)\n", - BASE_JD_EVENT_DRV_TERMINATED); - return 0; - } - - /* normal event processing */ - atomic_dec(&kctx->event_count); - atom = list_entry(kctx->event_list.next, struct kbase_jd_atom, dep_item[0]); - list_del(kctx->event_list.next); - - mutex_unlock(&kctx->event_mutex); - - dev_dbg(kctx->kbdev->dev, "event dequeuing %pK\n", (void *)atom); - uevent->event_code = atom->event_code; - - uevent->atom_number = (atom - kctx->jctx.atoms); - - if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) - kbase_jd_free_external_resources(atom); - - mutex_lock(&kctx->jctx.lock); - uevent->udata = kbase_event_process(kctx, atom); - mutex_unlock(&kctx->jctx.lock); - - return 0; -} - -KBASE_EXPORT_TEST_API(kbase_event_dequeue); - -/** - * kbase_event_process_noreport_worker - Worker for processing atoms that do not - * return an event but do have external - * resources - * @data: Work structure - */ -static void kbase_event_process_noreport_worker(struct work_struct *data) -{ - struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); - struct kbase_context *kctx = katom->kctx; - - if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) - kbase_jd_free_external_resources(katom); - - mutex_lock(&kctx->jctx.lock); - kbase_event_process(kctx, katom); - mutex_unlock(&kctx->jctx.lock); -} - -/** - * kbase_event_process_noreport - Process atoms that do not return an event - * @kctx: Context pointer - * @katom: Atom to be processed - * - * Atoms that do not have external resources will be processed immediately. - * Atoms that do have external resources will be processed on a workqueue, in - * order to avoid locking issues. - */ -static void kbase_event_process_noreport(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { - INIT_WORK(&katom->work, kbase_event_process_noreport_worker); - queue_work(kctx->event_workq, &katom->work); - } else { - kbase_event_process(kctx, katom); - } -} - -/** - * kbase_event_coalesce - Move pending events to the main event list - * @kctx: Context pointer - * - * kctx->event_list and kctx->event_coalesce_count must be protected - * by a lock unless this is the last thread using them - * (and we're about to terminate the lock). - * - * Return: The number of pending events moved to the main event list - */ -static int kbase_event_coalesce(struct kbase_context *kctx) -{ - const int event_count = kctx->event_coalesce_count; - - /* Join the list of pending events onto the tail of the main list - * and reset it - */ - list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); - kctx->event_coalesce_count = 0; - - /* Return the number of events moved */ - return event_count; -} - -void kbase_event_post(struct kbase_context *kctx, struct kbase_jd_atom *atom) -{ - struct kbase_device *kbdev = kctx->kbdev; - - dev_dbg(kbdev->dev, "Posting event for atom %pK\n", (void *)atom); - - if (WARN_ON(atom->status != KBASE_JD_ATOM_STATE_COMPLETED)) { - dev_warn(kbdev->dev, "%s: Atom %d (%pK) not completed (status %d)\n", __func__, - kbase_jd_atom_id(atom->kctx, atom), atom->kctx, atom->status); - return; - } - - if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) { - if (atom->event_code == BASE_JD_EVENT_DONE) { - dev_dbg(kbdev->dev, "Suppressing event (atom done)\n"); - kbase_event_process_noreport(kctx, atom); - return; - } - } - - if (atom->core_req & BASEP_JD_REQ_EVENT_NEVER) { - dev_dbg(kbdev->dev, "Suppressing event (never)\n"); - kbase_event_process_noreport(kctx, atom); - return; - } - KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, atom, TL_ATOM_STATE_POSTED); - if (atom->core_req & BASE_JD_REQ_EVENT_COALESCE) { - /* Don't report the event until other event(s) have completed */ - dev_dbg(kbdev->dev, "Deferring event (coalesced)\n"); - mutex_lock(&kctx->event_mutex); - list_add_tail(&atom->dep_item[0], &kctx->event_coalesce_list); - ++kctx->event_coalesce_count; - mutex_unlock(&kctx->event_mutex); - } else { - /* Report the event and any pending events now */ - int event_count = 1; - - mutex_lock(&kctx->event_mutex); - event_count += kbase_event_coalesce(kctx); - list_add_tail(&atom->dep_item[0], &kctx->event_list); - atomic_add(event_count, &kctx->event_count); - mutex_unlock(&kctx->event_mutex); - dev_dbg(kbdev->dev, "Reporting %d events\n", event_count); - - kbase_event_wakeup(kctx); - - /* Post-completion latency */ - trace_sysgraph(SGR_POST, kctx->id, kbase_jd_atom_id(kctx, atom)); - } -} -KBASE_EXPORT_TEST_API(kbase_event_post); - -void kbase_event_close(struct kbase_context *kctx) -{ - mutex_lock(&kctx->event_mutex); - atomic_set(&kctx->event_closed, true); - mutex_unlock(&kctx->event_mutex); - kbase_event_wakeup(kctx); -} - -int kbase_event_init(struct kbase_context *kctx) -{ - KBASE_DEBUG_ASSERT(kctx); - - INIT_LIST_HEAD(&kctx->event_list); - INIT_LIST_HEAD(&kctx->event_coalesce_list); - mutex_init(&kctx->event_mutex); - kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); - - if (kctx->event_workq == NULL) - return -EINVAL; - - return 0; -} - -KBASE_EXPORT_TEST_API(kbase_event_init); - -void kbase_event_cleanup(struct kbase_context *kctx) -{ - int event_count; - - KBASE_DEBUG_ASSERT(kctx); - KBASE_DEBUG_ASSERT(kctx->event_workq); - - flush_workqueue(kctx->event_workq); - destroy_workqueue(kctx->event_workq); - - /* We use kbase_event_dequeue to remove the remaining events as that - * deals with all the cleanup needed for the atoms. - * - * Note: use of kctx->event_list without a lock is safe because this must be the last - * thread using it (because we're about to terminate the lock) - */ - event_count = kbase_event_coalesce(kctx); - atomic_add(event_count, &kctx->event_count); - - while (!list_empty(&kctx->event_list)) { - struct base_jd_event_v2 event; - - kbase_event_dequeue(kctx, &event); - } -} - -KBASE_EXPORT_TEST_API(kbase_event_cleanup); diff --git a/drivers/gpu/arm/valhall/mali_kbase_fence.c b/drivers/gpu/arm/valhall/mali_kbase_fence.c deleted file mode 100644 index 8b0505a800a4..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_fence.c +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include -#include -#include - -#include - -/* Spin lock protecting all Mali fences as fence->lock. */ -static DEFINE_SPINLOCK(kbase_fence_lock); - -struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom) -{ - struct dma_fence *fence; - - WARN_ON(katom->dma_fence.fence); - - fence = kzalloc(sizeof(*fence), GFP_KERNEL); - if (!fence) - return NULL; - - dma_fence_init(fence, &kbase_fence_ops, &kbase_fence_lock, katom->dma_fence.context, - (u64)atomic_inc_return(&katom->dma_fence.seqno)); - - katom->dma_fence.fence = fence; - - return fence; -} diff --git a/drivers/gpu/arm/valhall/mali_kbase_fence.h b/drivers/gpu/arm/valhall/mali_kbase_fence.h index 9fe6c67b464c..f15f4048e922 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_fence.h +++ b/drivers/gpu/arm/valhall/mali_kbase_fence.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -34,7 +34,6 @@ #include #include -#if MALI_USE_CSF /* Number of digits needed to express the max value of given unsigned type. * * Details: The number of digits needed to express the max value of given type is log10(t_max) + 1 @@ -91,12 +90,13 @@ struct kbase_kcpu_dma_fence_meta { * * @base: Fence object itself. * @metadata: Pointer to metadata structure. + * @module: Pointer to the mali_kbase kernel module. */ struct kbase_kcpu_dma_fence { struct dma_fence base; struct kbase_kcpu_dma_fence_meta *metadata; + struct module *module; }; -#endif extern const struct dma_fence_ops kbase_fence_ops; @@ -123,86 +123,6 @@ struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); } while (0) #endif -#if !MALI_USE_CSF -/** - * kbase_fence_out_remove() - Removes the output fence from atom - * @katom: Atom to remove output fence for - * - * This will also release the reference to this fence which the atom keeps - */ -static inline void kbase_fence_out_remove(struct kbase_jd_atom *katom) -{ - if (katom->dma_fence.fence) { - dma_fence_put(katom->dma_fence.fence); - katom->dma_fence.fence = NULL; - } -} - -#if IS_ENABLED(CONFIG_SYNC_FILE) -/** - * kbase_fence_in_remove() - Removes the input fence from atom - * @katom: Atom to remove input fence for - * - * This will also release the reference to this fence which the atom keeps - */ -static inline void kbase_fence_in_remove(struct kbase_jd_atom *katom) -{ - if (katom->dma_fence.fence_in) { - dma_fence_put(katom->dma_fence.fence_in); - katom->dma_fence.fence_in = NULL; - } -} -#endif - -/** - * kbase_fence_out_is_ours() - Check if atom has a valid fence created by us - * @katom: Atom to check output fence for - * - * Return: true if fence exists and is valid, otherwise false - */ -static inline bool kbase_fence_out_is_ours(struct kbase_jd_atom *katom) -{ - return katom->dma_fence.fence && katom->dma_fence.fence->ops == &kbase_fence_ops; -} - -/** - * kbase_fence_out_signal() - Signal output fence of atom - * @katom: Atom to signal output fence for - * @status: Status to signal with (0 for success, < 0 for error) - * - * Return: 0 on success, < 0 on error - */ -static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, int status) -{ - if (status) - dma_fence_set_error_helper(katom->dma_fence.fence, status); - return dma_fence_signal(katom->dma_fence.fence); -} - -#if IS_ENABLED(CONFIG_SYNC_FILE) -/** - * kbase_fence_in_get() - Retrieve input fence for atom. - * @katom: Atom to get input fence from - * - * A ref will be taken for the fence, so use @kbase_fence_put() to release it - * - * Return: The fence, or NULL if there is no input fence for atom - */ -#define kbase_fence_in_get(katom) dma_fence_get((katom)->dma_fence.fence_in) -#endif - -/** - * kbase_fence_out_get() - Retrieve output fence for atom. - * @katom: Atom to get output fence from - * - * A ref will be taken for the fence, so use @kbase_fence_put() to release it - * - * Return: The fence, or NULL if there is no output fence for atom - */ -#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) - -#endif /* !MALI_USE_CSF */ - /** * kbase_fence_get() - Retrieve fence for a KCPUQ fence command. * @fence_info: KCPUQ fence command @@ -213,7 +133,6 @@ static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, int status */ #define kbase_fence_get(fence_info) dma_fence_get((fence_info)->fence) -#if MALI_USE_CSF static inline struct kbase_kcpu_dma_fence *kbase_kcpu_dma_fence_get(struct dma_fence *fence) { if (fence->ops == &kbase_fence_ops) @@ -237,7 +156,6 @@ static inline void kbase_kcpu_dma_fence_put(struct dma_fence *fence) if (kcpu_fence) kbase_kcpu_dma_fence_meta_put(kcpu_fence->metadata); } -#endif /* MALI_USE_CSF */ /** * kbase_fence_put() - Releases a reference to a fence diff --git a/drivers/gpu/arm/valhall/mali_kbase_fence_ops.c b/drivers/gpu/arm/valhall/mali_kbase_fence_ops.c index b10e69cea565..faaf4f1a82dc 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_fence_ops.c +++ b/drivers/gpu/arm/valhall/mali_kbase_fence_ops.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -31,21 +31,12 @@ static const char *kbase_fence_get_driver_name(struct dma_fence *fence) return KBASE_DRV_NAME; } -#if MALI_USE_CSF static const char *kbase_fence_get_timeline_name(struct dma_fence *fence) { struct kbase_kcpu_dma_fence *kcpu_fence = (struct kbase_kcpu_dma_fence *)fence; return kcpu_fence->metadata->timeline_name; } -#else -static const char *kbase_fence_get_timeline_name(struct dma_fence *fence) -{ - CSTD_UNUSED(fence); - - return KBASE_TIMELINE_NAME; -} -#endif /* MALI_USE_CSF */ static bool kbase_fence_enable_signaling(struct dma_fence *fence) { @@ -67,12 +58,13 @@ static void kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int pr_err("Fail to encode fence seqno to string"); } -#if MALI_USE_CSF static void kbase_fence_release(struct dma_fence *fence) { struct kbase_kcpu_dma_fence *kcpu_fence = (struct kbase_kcpu_dma_fence *)fence; kbase_kcpu_dma_fence_meta_put(kcpu_fence->metadata); + if (likely(kcpu_fence->module)) + module_put(kcpu_fence->module); kfree(kcpu_fence); } @@ -83,13 +75,5 @@ const struct dma_fence_ops kbase_fence_ops = { .wait = dma_fence_default_wait, .enable_signaling = kbase_fence_enable_signaling, .fence_value_str = kbase_fence_fence_value_str, .release = kbase_fence_release }; -#else -extern const struct dma_fence_ops kbase_fence_ops; /* silence checker warning */ -const struct dma_fence_ops kbase_fence_ops = { .wait = dma_fence_default_wait, - .get_driver_name = kbase_fence_get_driver_name, - .get_timeline_name = kbase_fence_get_timeline_name, - .enable_signaling = kbase_fence_enable_signaling, - .fence_value_str = kbase_fence_fence_value_str }; -#endif /* MALI_USE_CSF */ KBASE_EXPORT_TEST_API(kbase_fence_ops); diff --git a/drivers/gpu/arm/valhall/mali_kbase_gpuprops.c b/drivers/gpu/arm/valhall/mali_kbase_gpuprops.c index 55a7a796c58a..fd73c86e35a6 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_gpuprops.c +++ b/drivers/gpu/arm/valhall/mali_kbase_gpuprops.c @@ -239,11 +239,7 @@ static int kbase_gpuprops_get_props(struct kbase_device *kbdev) else gpu_props->max_threads = regdump->thread_max_threads; -#if MALI_USE_CSF gpu_props->impl_tech = KBASE_UBFX32(regdump->thread_features, 22U, 2); -#else /* MALI_USE_CSF */ - gpu_props->impl_tech = KBASE_UBFX32(regdump->thread_features, 30U, 2); -#endif /* MALI_USE_CSF */ if (IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI)) gpu_props->impl_tech = THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_NO_MALI; @@ -678,9 +674,6 @@ static void kbase_populate_user_data(struct kbase_device *kbdev, struct gpu_prop * GPUs like tTIx have additional fields like LSC_SIZE that are * otherwise reserved/RAZ on older GPUs. */ -#if !MALI_USE_CSF - data->core_props.num_exec_engines = KBASE_UBFX64(regdump->core_features, 0, 4); -#endif data->l2_props.log2_cache_size = KBASE_UBFX64(regdump->l2_features, 16U, 8); data->coherency_info.coherency = regdump->mem_features; @@ -707,15 +700,9 @@ static void kbase_populate_user_data(struct kbase_device *kbdev, struct gpu_prop else data->thread_props.tls_alloc = regdump->thread_tls_alloc; -#if MALI_USE_CSF data->thread_props.max_registers = KBASE_UBFX32(regdump->thread_features, 0U, 22); data->thread_props.max_task_queue = KBASE_UBFX32(regdump->thread_features, 24U, 8); data->thread_props.max_thread_group_split = 0; -#else - data->thread_props.max_registers = KBASE_UBFX32(regdump->thread_features, 0U, 16); - data->thread_props.max_task_queue = KBASE_UBFX32(regdump->thread_features, 16U, 8); - data->thread_props.max_thread_group_split = KBASE_UBFX32(regdump->thread_features, 24U, 6); -#endif if (data->thread_props.max_registers == 0) { data->thread_props.max_registers = THREAD_MR_DEFAULT; diff --git a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_defs.h index 96aeb6188a05..fba612771e33 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_defs.h +++ b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -40,10 +40,6 @@ * @backend: GPU backend specific data for HW access layer */ struct kbase_hwaccess_data { -#if !MALI_USE_CSF - struct kbase_context *active_kctx[BASE_JM_MAX_NR_SLOTS]; -#endif - struct kbase_backend_data backend; }; diff --git a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_jm.h deleted file mode 100644 index ed91019ff74c..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_jm.h +++ /dev/null @@ -1,308 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * HW access job manager common APIs - */ - -#ifndef _KBASE_HWACCESS_JM_H_ -#define _KBASE_HWACCESS_JM_H_ - -/** - * kbase_backend_run_atom() - Run an atom on the GPU - * @kbdev: Device pointer - * @katom: Atom to run - * - * Caller must hold the HW access lock - */ -void kbase_backend_run_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -/** - * kbase_backend_slot_update - Update state based on slot ringbuffers - * - * @kbdev: Device pointer - * - * Inspect the jobs in the slot ringbuffers and update state. - * - * This will cause jobs to be submitted to hardware if they are unblocked - */ -void kbase_backend_slot_update(struct kbase_device *kbdev); - -/** - * kbase_backend_find_and_release_free_address_space() - Release a free AS - * @kbdev: Device pointer - * @kctx: Context pointer - * - * This function can evict an idle context from the runpool, freeing up the - * address space it was using. - * - * The address space is marked as in use. The caller must either assign a - * context using kbase_gpu_use_ctx(), or release it using - * kbase_ctx_sched_release() - * - * Return: Number of free address space, or KBASEP_AS_NR_INVALID if none - * available - */ -int kbase_backend_find_and_release_free_address_space(struct kbase_device *kbdev, - struct kbase_context *kctx); - -/** - * kbase_backend_use_ctx() - Activate a currently unscheduled context, using the - * provided address space. - * @kbdev: Device pointer - * @kctx: Context pointer. May be NULL - * @as_nr: Free address space to use - * - * kbase_gpu_next_job() will pull atoms from the active context. - * - * Return: true if successful, false if ASID not assigned. - */ -bool kbase_backend_use_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, int as_nr); - -/** - * kbase_backend_use_ctx_sched() - Activate a context. - * @kbdev: Device pointer - * @kctx: Context pointer - * @js: Job slot to activate context on - * - * kbase_gpu_next_job() will pull atoms from the active context. - * - * The context must already be scheduled and assigned to an address space. If - * the context is not scheduled, then kbase_gpu_use_ctx() should be used - * instead. - * - * Caller must hold hwaccess_lock - * - * Return: true if context is now active, false otherwise (ie if context does - * not have an address space assigned) - */ -bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js); - -/** - * kbase_backend_release_ctx_irq - Release a context from the GPU. This will - * de-assign the assigned address space. - * @kbdev: Device pointer - * @kctx: Context pointer - * - * Caller must hold kbase_device->mmu_hw_mutex and hwaccess_lock - */ -void kbase_backend_release_ctx_irq(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbase_backend_release_ctx_noirq - Release a context from the GPU. This will - * de-assign the assigned address space. - * @kbdev: Device pointer - * @kctx: Context pointer - * - * Caller must hold kbase_device->mmu_hw_mutex - * - * This function must perform any operations that could not be performed in IRQ - * context by kbase_backend_release_ctx_irq(). - */ -void kbase_backend_release_ctx_noirq(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbase_backend_cache_clean - Perform a cache clean if the given atom requires - * one - * @kbdev: Device pointer - * @katom: Pointer to the failed atom - * - * On some GPUs, the GPU cache must be cleaned following a failed atom. This - * function performs a clean if it is required by @katom. - */ -void kbase_backend_cache_clean(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -/** - * kbase_backend_complete_wq() - Perform backend-specific actions required on - * completing an atom. - * @kbdev: Device pointer - * @katom: Pointer to the atom to complete - * - * This function should only be called from kbase_jd_done_worker() or - * js_return_worker(). - * - * Return: true if atom has completed, false if atom should be re-submitted - */ -void kbase_backend_complete_wq(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -#if !MALI_USE_CSF -/** - * kbase_backend_complete_wq_post_sched - Perform backend-specific actions - * required on completing an atom, after - * any scheduling has taken place. - * @kbdev: Device pointer - * @core_req: Core requirements of atom - * - * This function should only be called from kbase_jd_done_worker() or - * js_return_worker(). - */ -void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, base_jd_core_req core_req); -#endif /* !MALI_USE_CSF */ - -/** - * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU - * and remove any others from the ringbuffers. - * @kbdev: Device pointer - * @end_timestamp: Timestamp of reset - */ -void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp); - -/** - * kbase_backend_inspect_tail - Return the atom currently at the tail of slot - * @js - * @kbdev: Device pointer - * @js: Job slot to inspect - * - * Return: Atom currently at the head of slot @js, or NULL - */ -struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, unsigned int js); - -/** - * kbase_backend_nr_atoms_on_slot() - Return the number of atoms currently on a - * slot. - * @kbdev: Device pointer - * @js: Job slot to inspect - * - * Return: Number of atoms currently on slot - */ -int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, unsigned int js); - -/** - * kbase_backend_nr_atoms_submitted() - Return the number of atoms on a slot - * that are currently on the GPU. - * @kbdev: Device pointer - * @js: Job slot to inspect - * - * Return: Number of atoms currently on slot @js that are currently on the GPU. - */ -u32 kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, unsigned int js); - -/** - * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs - * has changed. - * @kbdev: Device pointer - * - * Perform any required backend-specific actions (eg starting/stopping - * scheduling timers). - */ -void kbase_backend_ctx_count_changed(struct kbase_device *kbdev); - -/** - * kbase_backend_timeouts_changed() - Job Scheduler timeouts have changed. - * @kbdev: Device pointer - * - * Perform any required backend-specific actions (eg updating timeouts of - * currently running atoms). - */ -void kbase_backend_timeouts_changed(struct kbase_device *kbdev); - -/** - * kbase_backend_slot_free() - Return the number of jobs that can be currently - * submitted to slot @js. - * @kbdev: Device pointer - * @js: Job slot to inspect - * - * Return: Number of jobs that can be submitted. - */ -int kbase_backend_slot_free(struct kbase_device *kbdev, unsigned int js); - -/** - * kbase_job_check_leave_disjoint - potentially leave disjoint state - * @kbdev: kbase device - * @target_katom: atom which is finishing - * - * Work out whether to leave disjoint state when finishing an atom that was - * originated by kbase_job_check_enter_disjoint(). - */ -void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, struct kbase_jd_atom *target_katom); - -/** - * kbase_backend_jm_kill_running_jobs_from_kctx - Kill all jobs that are - * currently running on GPU from a context - * @kctx: Context pointer - * - * This is used in response to a page fault to remove all jobs from the faulting - * context from the hardware. - * - * Caller must hold hwaccess_lock. - */ -void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx); - -/** - * kbase_jm_wait_for_zero_jobs - Wait for context to have zero jobs running, and - * to be descheduled. - * @kctx: Context pointer - * - * This should be called following kbase_js_zap_context(), to ensure the context - * can be safely destroyed. - */ -void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx); - -/** - * kbase_backend_get_current_flush_id - Return the current flush ID - * - * @kbdev: Device pointer - * - * Return: the current flush ID to be recorded for each job chain - */ -u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev); - -/** - * kbase_job_slot_hardstop - Hard-stop the specified job slot - * @kctx: The kbase context that contains the job(s) that should - * be hard-stopped - * @js: The job slot to hard-stop - * @target_katom: The job that should be hard-stopped (or NULL for all - * jobs from the context) - * Context: - * The job slot lock must be held when calling this function. - */ -void kbase_job_slot_hardstop(struct kbase_context *kctx, unsigned int js, - struct kbase_jd_atom *target_katom); - -/** - * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms - * currently on the GPU - * @kbdev: Device pointer - * - * Return: true if there are any atoms on the GPU, false otherwise - */ -bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev); - -/** - * kbase_backend_slot_kctx_purge_locked - Perform a purge on the slot_rb tracked - * kctx - * - * @kbdev: Device pointer - * @kctx: The kbase context that needs to be purged from slot_rb[] - * - * For JM GPUs, the L1 read only caches may need a start_flush invalidation, - * potentially on all slots (even if the kctx was only using a single slot), - * following a context termination or address-space ID recycle. This function - * performs a clean-up purge on the given kctx which if it has been tracked by - * slot_rb[] objects. - * - * Caller must hold kbase_device->hwaccess_lock. - */ -void kbase_backend_slot_kctx_purge_locked(struct kbase_device *kbdev, struct kbase_context *kctx); - -#endif /* _KBASE_HWACCESS_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_pm.h index 982547d16022..a396f02b59ed 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_pm.h +++ b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_pm.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -114,7 +114,6 @@ void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); */ void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); -#if MALI_USE_CSF /** * kbase_pm_set_debug_core_mask - Set the debug core mask. * @@ -124,20 +123,6 @@ void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); * This determines which cores the power manager is allowed to use. */ void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask); -#else -/** - * kbase_pm_set_debug_core_mask - Set the debug core mask. - * - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * @new_core_mask: The core mask to use, as an array where each element refers - * to a job slot. - * @new_core_mask_size: Number of elements in the core mask array. - * - * This determines which cores the power manager is allowed to use. - */ -void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 *new_core_mask, - size_t new_core_mask_size); -#endif /* MALI_USE_CSF */ /** * kbase_pm_ca_get_policy - Get the current policy. diff --git a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_time.h index 222ff2001e56..2df4cad514f8 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_hwaccess_time.h +++ b/drivers/gpu/arm/valhall/mali_kbase_hwaccess_time.h @@ -53,15 +53,12 @@ * */ struct kbase_backend_time { -#if MALI_USE_CSF u64 multiplier; u64 divisor; s64 gpu_timestamp_offset; -#endif unsigned int device_scaled_timeouts[KBASE_TIMEOUT_SELECTOR_COUNT]; }; -#if MALI_USE_CSF /** * kbase_backend_time_convert_gpu_to_cpu() - Convert GPU timestamp to CPU timestamp. * @@ -105,7 +102,6 @@ void kbase_backend_invalidate_gpu_timestamp_offset(struct kbase_device *kbdev); */ u64 kbase_backend_read_gpu_timestamp_offset_reg(struct kbase_device *kbdev); #endif -#endif /** * kbase_backend_get_gpu_time() - Get current GPU time @@ -205,4 +201,11 @@ u64 kbase_arch_timer_get_cntfrq(struct kbase_device *kbdev); */ int kbase_backend_time_init(struct kbase_device *kbdev); +/** + * kbase_gpu_timestamp_offset_debugfs_init - Define debugfs entry to read the GPU timestamp offset. + * + * @kbdev: kbase device pointer + * + */ +void kbase_gpu_timestamp_offset_debugfs_init(struct kbase_device *kbdev); #endif /* _KBASE_BACKEND_TIME_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_io.c b/drivers/gpu/arm/valhall/mali_kbase_io.c index 33acab795f75..49e1ea5175d6 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_io.c +++ b/drivers/gpu/arm/valhall/mali_kbase_io.c @@ -36,25 +36,6 @@ struct kbase_io { struct kbase_device *kbdev; }; -/** - * kbase_io_is_gpu_removed() - Has the GPU been removed. - * @kbdev: Kbase device pointer - * - * When Kbase takes too long to give up the GPU, the Arbiter - * can remove it. This will then be followed by a GPU lost event. - * This function will return true if the GPU has been removed. - * When this happens register reads will be zero. A zero GPU_ID is - * invalid so this is used to detect when GPU is removed. - * - * Return: True if GPU removed - */ -static bool kbase_io_is_gpu_removed(struct kbase_device *kbdev) -{ - if (!kbase_has_arbiter(kbdev)) - return false; - return (KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_ID)) == 0); -} - void kbase_io_set_status(struct kbase_io *io, enum kbase_io_status_bits status_bit) { set_bit(status_bit, io->status); @@ -81,21 +62,13 @@ KBASE_EXPORT_TEST_API(kbase_io_is_gpu_powered); bool kbase_io_is_aw_removed(struct kbase_device *kbdev) { - return (kbdev->arb.arb_if && test_bit(KBASE_IO_STATUS_AW_REMOVED, kbdev->io->status)); + return test_bit(KBASE_IO_STATUS_AW_REMOVED, kbdev->io->status); } KBASE_EXPORT_TEST_API(kbase_io_is_aw_removed); bool kbase_io_has_gpu(struct kbase_device *kbdev) { - if (kbase_io_test_status(kbdev, KBASE_IO_STATUS_AW_REMOVED)) - return false; - - if (kbase_io_is_gpu_removed(kbdev)) { - kbase_io_set_status(kbdev->io, KBASE_IO_STATUS_AW_REMOVED); - return false; - } - - return true; + return kbase_io_is_gpu_powered(kbdev) && !kbase_io_is_aw_removed(kbdev); } KBASE_EXPORT_TEST_API(kbase_io_has_gpu); diff --git a/drivers/gpu/arm/valhall/mali_kbase_io.h b/drivers/gpu/arm/valhall/mali_kbase_io.h index 3a352fcb92b4..0f1cf8a6264b 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_io.h +++ b/drivers/gpu/arm/valhall/mali_kbase_io.h @@ -32,14 +32,12 @@ struct kbase_io; /** * enum kbase_io_status_bits - Status bits for kbase I/O interface. * - * @KBASE_IO_STATUS_GPU_SUSPENDED: The GPU is suspended. * @KBASE_IO_STATUS_GPU_OFF: The GPU is OFF. * @KBASE_IO_STATUS_AW_REMOVED: The GPU access window has been removed. * @KBASE_IO_STATUS_NUM_BITS: Number of bits used to encode the status. */ enum kbase_io_status_bits { - KBASE_IO_STATUS_GPU_SUSPENDED = 0, - KBASE_IO_STATUS_GPU_OFF, + KBASE_IO_STATUS_GPU_OFF = 0, KBASE_IO_STATUS_AW_REMOVED, KBASE_IO_STATUS_NUM_BITS, }; diff --git a/drivers/gpu/arm/valhall/mali_kbase_ioctl_helpers.h b/drivers/gpu/arm/valhall/mali_kbase_ioctl_helpers.h index d8d144c46f16..cfecc2de6b92 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_ioctl_helpers.h +++ b/drivers/gpu/arm/valhall/mali_kbase_ioctl_helpers.h @@ -286,8 +286,6 @@ check_padding_KBASE_IOCTL_KINSTR_PRFCNT_SETUP(union kbase_ioctl_kinstr_prfcnt_se #if MALI_UNIT_TEST #endif /* MALI_UNIT_TEST */ -#if MALI_USE_CSF - static inline int check_padding_KBASE_IOCTL_CS_QUEUE_REGISTER(struct kbase_ioctl_cs_queue_register *p) { @@ -518,31 +516,4 @@ check_padding_KBASE_IOCTL_CS_TILER_HEAP_SIZE(union kbase_ioctl_cs_tiler_heap_siz return 0; } -#else /* MALI_USE_CSF */ - -static inline int check_padding_KBASE_IOCTL_JOB_SUBMIT(struct kbase_ioctl_job_submit *p) -{ - return 0; -} - -static inline int -check_padding_KBASE_IOCTL_SOFT_EVENT_UPDATE(struct kbase_ioctl_soft_event_update *p) -{ - return 0; -} - -static inline int check_padding_KBASE_IOCTL_KINSTR_JM_FD(union kbase_kinstr_jm_fd *p) -{ - size_t i; - - for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) { - if (p->in.padding[i]) - return -1; - } - - return 0; -} - -#endif /* !MALI_USE_CSF */ - #endif /* _KBASE_IOCTL_HELPERS_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_jd.c b/drivers/gpu/arm/valhall/mali_kbase_jd.c deleted file mode 100644 index e56bc60ed8c3..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_jd.c +++ /dev/null @@ -1,1492 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#if IS_ENABLED(CONFIG_COMPAT) -#include -#endif -#include -#include -#include -#include -#include -#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE -#include -#else -#include -#endif - -#include -#include -#include -#include -#include -#include - -/* Return whether katom will run on the GPU or not. Currently only soft jobs and - * dependency-only atoms do not run on the GPU - */ -#define IS_GPU_ATOM(katom) \ - (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ - ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP))) - -/* - * This is the kernel side of the API. Only entry points are: - * - kbase_jd_submit(): Called from userspace to submit a single bag - * - kbase_jd_done(): Called from interrupt context to track the - * completion of a job. - * Callouts: - * - to the job manager (enqueue a job) - * - to the event subsystem (signals the completion/failure of bag/job-chains). - */ - -static void __user *get_compat_pointer(struct kbase_context *kctx, const u64 p) -{ -#if IS_ENABLED(CONFIG_COMPAT) - if (kbase_ctx_flag(kctx, KCTX_COMPAT)) - return compat_ptr(p); -#endif - return u64_to_user_ptr(p); -} - -/* Mark an atom as complete, and trace it in kinstr_jm */ -static void jd_mark_atom_complete(struct kbase_jd_atom *katom) -{ - katom->status = KBASE_JD_ATOM_STATE_COMPLETED; - kbase_kinstr_jm_atom_complete(katom); - dev_dbg(katom->kctx->kbdev->dev, "Atom %pK status to completed\n", (void *)katom); - KBASE_TLSTREAM_TL_JD_ATOM_COMPLETE(katom->kctx->kbdev, katom); -} - -/* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs - * - * Returns whether the JS needs a reschedule. - * - * Note that the caller must also check the atom status and - * if it is KBASE_JD_ATOM_STATE_COMPLETED must call kbase_jd_done_nolock - */ -static bool jd_run_atom(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - - dev_dbg(kctx->kbdev->dev, "JD run atom %pK in kctx %pK\n", (void *)katom, (void *)kctx); - - KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); - - if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { - /* Dependency only atom */ - trace_sysgraph(SGR_SUBMIT, kctx->id, kbase_jd_atom_id(katom->kctx, katom)); - jd_mark_atom_complete(katom); - return false; - } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { - /* Soft-job */ - if (katom->will_fail_event_code) { - kbase_finish_soft_job(katom); - jd_mark_atom_complete(katom); - return false; - } - if (kbase_process_soft_job(katom) == 0) { - kbase_finish_soft_job(katom); - jd_mark_atom_complete(katom); - } - return false; - } - - katom->status = KBASE_JD_ATOM_STATE_IN_JS; - dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n", (void *)katom); - /* Queue an action about whether we should try scheduling a context */ - return kbasep_js_add_job(kctx, katom); -} - -void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom) -{ - struct kbase_device *kbdev; - - KBASE_DEBUG_ASSERT(katom); - kbdev = katom->kctx->kbdev; - KBASE_DEBUG_ASSERT(kbdev); - - /* Check whether the atom's other dependencies were already met. If - * katom is a GPU atom then the job scheduler may be able to represent - * the dependencies, hence we may attempt to submit it before they are - * met. Other atoms must have had both dependencies resolved. - */ - if (IS_GPU_ATOM(katom) || (!kbase_jd_katom_dep_atom(&katom->dep[0]) && - !kbase_jd_katom_dep_atom(&katom->dep[1]))) { - /* katom dep complete, attempt to run it */ - bool resched = false; - - KBASE_TLSTREAM_TL_RUN_ATOM_START(katom->kctx->kbdev, katom, - kbase_jd_atom_id(katom->kctx, katom)); - resched = jd_run_atom(katom); - KBASE_TLSTREAM_TL_RUN_ATOM_END(katom->kctx->kbdev, katom, - kbase_jd_atom_id(katom->kctx, katom)); - - if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) { - /* The atom has already finished */ - resched |= kbase_jd_done_nolock(katom, true); - } - - if (resched) - kbase_js_sched_all(kbdev); - } -} - -void kbase_jd_free_external_resources(struct kbase_jd_atom *katom) -{ - CSTD_UNUSED(katom); -} - -static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) -{ - KBASE_DEBUG_ASSERT(katom); - KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); - - kbase_gpu_vm_lock(katom->kctx); - /* only roll back if extres is non-NULL */ - if (katom->extres) { - u32 res_no; - - res_no = katom->nr_extres; - while (res_no-- > 0) { - kbase_unmap_external_resource(katom->kctx, katom->extres[res_no]); - } - kfree(katom->extres); - katom->extres = NULL; - } - kbase_gpu_vm_unlock(katom->kctx); -} - -/* - * Set up external resources needed by this job. - * - * jctx.lock must be held when this is called. - */ - -static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, - const struct base_jd_atom *user_atom) -{ - int err = -EINVAL; - u32 res_no; - struct base_external_resource *input_extres; - size_t copy_size; - - KBASE_DEBUG_ASSERT(katom); - KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); - - /* no resources encoded, early out */ - if (!katom->nr_extres) - return -EINVAL; - - katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL); - if (!katom->extres) - return -ENOMEM; - - input_extres = kmalloc_array(katom->nr_extres, sizeof(*input_extres), GFP_KERNEL); - if (!input_extres) { - err = -ENOMEM; - goto failed_input_alloc; - } - - if (check_mul_overflow(sizeof(*input_extres), (size_t)katom->nr_extres, ©_size)) { - err = -EINVAL; - goto failed_input_copy; - } - - if (copy_from_user(input_extres, get_compat_pointer(katom->kctx, user_atom->extres_list), - copy_size) != 0) { - err = -EINVAL; - goto failed_input_copy; - } - - /* Take the processes mmap lock */ - down_read(kbase_mem_get_process_mmap_lock()); - - /* need to keep the GPU VM locked while we set up UMM buffers */ - kbase_gpu_vm_lock(katom->kctx); - for (res_no = 0; res_no < katom->nr_extres; res_no++) { - struct base_external_resource *user_res = &input_extres[res_no]; - struct kbase_va_region *reg; - - reg = kbase_region_tracker_find_region_enclosing_address( - katom->kctx, - user_res->ext_resource & ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE); - /* did we find a matching region object? */ - if (unlikely(kbase_is_region_invalid_or_free(reg))) { - /* roll back */ - goto failed_loop; - } - - if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) && - (reg->flags & KBASE_REG_PROTECTED)) { - katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED; - } - - err = kbase_map_external_resource(katom->kctx, reg, current->mm); - if (err) - goto failed_loop; - - katom->extres[res_no] = reg; - } - /* successfully parsed the extres array */ - /* drop the vm lock now */ - kbase_gpu_vm_unlock(katom->kctx); - - /* Release the processes mmap lock */ - up_read(kbase_mem_get_process_mmap_lock()); - - /* Free the buffer holding data from userspace */ - kfree(input_extres); - - /* all done OK */ - return 0; - -/* error handling section */ -failed_loop: - /* undo the loop work. We are guaranteed to have access to the VA region - * as we hold a reference to it until it's unmapped - */ - while (res_no-- > 0) { - struct kbase_va_region *reg = katom->extres[res_no]; - - kbase_unmap_external_resource(katom->kctx, reg); - } - kbase_gpu_vm_unlock(katom->kctx); - - /* Release the processes mmap lock */ - up_read(kbase_mem_get_process_mmap_lock()); - -failed_input_copy: - kfree(input_extres); -failed_input_alloc: - kfree(katom->extres); - katom->extres = NULL; - return err; -} - -static inline void jd_resolve_dep(struct list_head *out_list, struct kbase_jd_atom *katom, u8 d, - bool ctx_is_dying) -{ - u8 other_d = !d; - - while (!list_empty(&katom->dep_head[d])) { - struct kbase_jd_atom *dep_atom; - struct kbase_jd_atom *other_dep_atom; - u8 dep_type; - - dep_atom = list_entry(katom->dep_head[d].next, struct kbase_jd_atom, dep_item[d]); - list_del(katom->dep_head[d].next); - - dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]); - kbase_jd_katom_dep_clear(&dep_atom->dep[d]); - - if (katom->event_code != BASE_JD_EVENT_DONE && - (dep_type != BASE_JD_DEP_TYPE_ORDER)) { - dep_atom->event_code = katom->event_code; - KBASE_DEBUG_ASSERT(dep_atom->status != KBASE_JD_ATOM_STATE_UNUSED); - - dep_atom->will_fail_event_code = dep_atom->event_code; - } - other_dep_atom = - (struct kbase_jd_atom *)kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]); - - if (!dep_atom->in_jd_list && - (!other_dep_atom || - (IS_GPU_ATOM(dep_atom) && !ctx_is_dying && !dep_atom->will_fail_event_code && - !other_dep_atom->will_fail_event_code))) { - dep_atom->in_jd_list = true; - list_add_tail(&dep_atom->jd_item, out_list); - } - } -} - -/** - * is_dep_valid - Validate that a dependency is valid for early dependency - * submission - * @katom: Dependency atom to validate - * - * A dependency is valid if any of the following are true : - * - It does not exist (a non-existent dependency does not block submission) - * - It is in the job scheduler - * - It has completed, does not have a failure event code, and has not been - * marked to fail in the future - * - * Return: true if valid, false otherwise - */ -static bool is_dep_valid(struct kbase_jd_atom *katom) -{ - /* If there's no dependency then this is 'valid' from the perspective of - * early dependency submission - */ - if (!katom) - return true; - - /* Dependency must have reached the job scheduler */ - if (katom->status < KBASE_JD_ATOM_STATE_IN_JS) - return false; - - /* If dependency has completed and has failed or will fail then it is - * not valid - */ - if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && - (katom->event_code != BASE_JD_EVENT_DONE || katom->will_fail_event_code)) - return false; - - return true; -} - -static void jd_try_submitting_deps(struct list_head *out_list, struct kbase_jd_atom *node) -{ - int i; - - for (i = 0; i < 2; i++) { - struct list_head *pos; - - list_for_each(pos, &node->dep_head[i]) { - struct kbase_jd_atom *dep_atom = - list_entry(pos, struct kbase_jd_atom, dep_item[i]); - - if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) { - /*Check if atom deps look sane*/ - bool dep0_valid = is_dep_valid(dep_atom->dep[0].atom); - bool dep1_valid = is_dep_valid(dep_atom->dep[1].atom); - - if (dep0_valid && dep1_valid) { - dep_atom->in_jd_list = true; - list_add(&dep_atom->jd_item, out_list); - } - } - } - } -} - -#if MALI_JIT_PRESSURE_LIMIT_BASE -/** - * jd_update_jit_usage - Update just-in-time physical memory usage for an atom. - * - * @katom: An atom that has just finished. - * - * Read back actual just-in-time memory region usage from atoms that provide - * this information, and update the current physical page pressure. - * - * The caller must hold the kbase_jd_context.lock. - */ -static void jd_update_jit_usage(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct kbase_va_region *reg; - struct kbase_vmap_struct mapping; - u64 *ptr; - u64 used_pages; - unsigned int idx; - - lockdep_assert_held(&kctx->jctx.lock); - - /* If this atom wrote to JIT memory, find out how much it has written - * and update the usage information in the region. - */ - for (idx = 0; idx < ARRAY_SIZE(katom->jit_ids) && katom->jit_ids[idx]; idx++) { - enum heap_pointer { LOW = 0, HIGH, COUNT }; - size_t size_to_read; - u64 read_val; - - reg = kctx->jit_alloc[katom->jit_ids[idx]]; - - if (!reg) { - dev_warn(kctx->kbdev->dev, "%s: JIT id[%u]=%u has no region\n", __func__, - idx, katom->jit_ids[idx]); - continue; - } - - if (reg == KBASE_RESERVED_REG_JIT_ALLOC) { - dev_warn(kctx->kbdev->dev, - "%s: JIT id[%u]=%u has failed to allocate a region\n", __func__, - idx, katom->jit_ids[idx]); - continue; - } - - if (!reg->heap_info_gpu_addr) - continue; - - size_to_read = sizeof(*ptr); - if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) - size_to_read = sizeof(u32); - else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) - size_to_read = sizeof(u64[COUNT]); - - ptr = kbase_vmap_prot(kctx, reg->heap_info_gpu_addr, size_to_read, KBASE_REG_CPU_RD, - &mapping); - - if (!ptr) { - dev_warn(kctx->kbdev->dev, - "%s: JIT id[%u]=%u start=0x%llx unable to map end marker %llx\n", - __func__, idx, katom->jit_ids[idx], reg->start_pfn << PAGE_SHIFT, - reg->heap_info_gpu_addr); - continue; - } - - if (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) { - read_val = READ_ONCE(*(u32 *)ptr); - used_pages = PFN_UP(read_val); - } else { - u64 addr_end; - - if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { - const unsigned long extension_bytes = reg->extension << PAGE_SHIFT; - const u64 low_ptr = ptr[LOW]; - const u64 high_ptr = ptr[HIGH]; - - /* As either the low or high pointer could - * consume their partition and move onto the - * next chunk, we need to account for both. - * In the case where nothing has been allocated - * from the high pointer the whole chunk could - * be backed unnecessarily - but the granularity - * is the chunk size anyway and any non-zero - * offset of low pointer from the start of the - * chunk would result in the whole chunk being - * backed. - */ - read_val = max(high_ptr, low_ptr); - - /* kbase_check_alloc_sizes() already satisfies - * this, but here to avoid future maintenance - * hazards - */ - WARN_ON(!is_power_of_2(extension_bytes)); - addr_end = ALIGN(read_val, extension_bytes); - } else { - addr_end = read_val = READ_ONCE(*ptr); - } - - if (addr_end >= (reg->start_pfn << PAGE_SHIFT)) - used_pages = PFN_UP(addr_end) - reg->start_pfn; - else - used_pages = reg->used_pages; - } - - trace_mali_jit_report(katom, reg, idx, read_val, used_pages); - kbase_trace_jit_report_gpu_mem(kctx, reg, 0u); - - /* We can never have used more pages than the VA size of the - * region - */ - if (used_pages > reg->nr_pages) { - dev_warn( - kctx->kbdev->dev, - "%s: JIT id[%u]=%u start=0x%llx used_pages %llx > %zx (read 0x%llx as %s%s)\n", - __func__, idx, katom->jit_ids[idx], reg->start_pfn << PAGE_SHIFT, - used_pages, reg->nr_pages, read_val, - (reg->flags & KBASE_REG_HEAP_INFO_IS_SIZE) ? "size" : "addr", - (reg->flags & KBASE_REG_TILER_ALIGN_TOP) ? " with align" : ""); - used_pages = reg->nr_pages; - } - /* Note: one real use case has an atom correctly reporting 0 - * pages in use. This happens in normal use-cases but may only - * happen for a few of the application's frames. - */ - - kbase_vunmap(kctx, &mapping); - - kbase_jit_report_update_pressure(kctx, reg, used_pages, 0u); - } - - kbase_jit_retry_pending_alloc(kctx); -} -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - -bool kbase_jd_done_nolock(struct kbase_jd_atom *katom, bool post_immediately) -{ - struct kbase_context *kctx = katom->kctx; - struct list_head completed_jobs; - struct list_head runnable_jobs; - bool need_to_try_schedule_context = false; - int i; - - lockdep_assert_held(&kctx->jctx.lock); - - KBASE_TLSTREAM_TL_JD_DONE_NO_LOCK_START(kctx->kbdev, katom); - - INIT_LIST_HEAD(&completed_jobs); - INIT_LIST_HEAD(&runnable_jobs); - - KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); - -#if MALI_JIT_PRESSURE_LIMIT_BASE - if (kbase_ctx_flag(kctx, KCTX_JPL_ENABLED)) - jd_update_jit_usage(katom); -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /* This is needed in case an atom is failed due to being invalid, this - * can happen *before* the jobs that the atom depends on have completed - */ - for (i = 0; i < 2; i++) { - if (kbase_jd_katom_dep_atom(&katom->dep[i])) { - list_del(&katom->dep_item[i]); - kbase_jd_katom_dep_clear(&katom->dep[i]); - } - } - - jd_mark_atom_complete(katom); - - list_add_tail(&katom->jd_item, &completed_jobs); - - while (!list_empty(&completed_jobs)) { - katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item); - list_del(completed_jobs.prev); - KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); - - for (i = 0; i < 2; i++) - jd_resolve_dep(&runnable_jobs, katom, i, kbase_ctx_flag(kctx, KCTX_DYING)); - - if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) - kbase_jd_post_external_resources(katom); - - while (!list_empty(&runnable_jobs)) { - struct kbase_jd_atom *node; - - node = list_entry(runnable_jobs.next, struct kbase_jd_atom, jd_item); - list_del(runnable_jobs.next); - node->in_jd_list = false; - - dev_dbg(kctx->kbdev->dev, "List node %pK has status %d\n", node, - node->status); - - KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); - if (node->status == KBASE_JD_ATOM_STATE_IN_JS) - continue; - - if (node->status != KBASE_JD_ATOM_STATE_COMPLETED && - !kbase_ctx_flag(kctx, KCTX_DYING)) { - KBASE_TLSTREAM_TL_RUN_ATOM_START(kctx->kbdev, node, - kbase_jd_atom_id(kctx, node)); - need_to_try_schedule_context |= jd_run_atom(node); - KBASE_TLSTREAM_TL_RUN_ATOM_END(kctx->kbdev, node, - kbase_jd_atom_id(kctx, node)); - } else { - node->event_code = katom->event_code; - - if (node->core_req & BASE_JD_REQ_SOFT_JOB) { - WARN_ON(!list_empty(&node->queue)); - kbase_finish_soft_job(node); - } - node->status = KBASE_JD_ATOM_STATE_COMPLETED; - } - - if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) { - list_add_tail(&node->jd_item, &completed_jobs); - } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS && - !node->will_fail_event_code) { - /* Node successfully submitted, try submitting - * dependencies as they may now be representable - * in JS - */ - jd_try_submitting_deps(&runnable_jobs, node); - } - } - - /* Register a completed job as a disjoint event when the GPU - * is in a disjoint state (ie. being reset). - */ - kbase_disjoint_event_potential(kctx->kbdev); - if (post_immediately && list_empty(&kctx->completed_jobs)) - kbase_event_post(kctx, katom); - else - list_add_tail(&katom->jd_item, &kctx->completed_jobs); - - /* Decrement and check the TOTAL number of jobs. This includes - * those not tracked by the scheduler: 'not ready to run' and - * 'dependency-only' jobs. - */ - if (--kctx->jctx.job_nr == 0) - /* All events are safely queued now, and we can signal - * any waiter that we've got no more jobs (so we can be - * safely terminated) - */ - wake_up(&kctx->jctx.zero_jobs_wait); - } - KBASE_TLSTREAM_TL_JD_DONE_NO_LOCK_END(kctx->kbdev, katom); - return need_to_try_schedule_context; -} -KBASE_EXPORT_TEST_API(kbase_jd_done_nolock); - -/* Trace an atom submission. */ -static void jd_trace_atom_submit(struct kbase_context *const kctx, - struct kbase_jd_atom *const katom, int *priority) -{ - struct kbase_device *const kbdev = kctx->kbdev; - - KBASE_TLSTREAM_TL_NEW_ATOM(kbdev, katom, (u32)kbase_jd_atom_id(kctx, katom)); - KBASE_TLSTREAM_TL_RET_ATOM_CTX(kbdev, katom, kctx); - if (priority) - KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(kbdev, katom, (u32)*priority); - KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_IDLE); - kbase_kinstr_jm_atom_queue(katom); -} - -static bool jd_submit_atom(struct kbase_context *const kctx, - const struct base_jd_atom *const user_atom, - struct kbase_jd_atom *const katom) -{ - struct kbase_device *kbdev = kctx->kbdev; - struct kbase_jd_context *jctx = &kctx->jctx; - int queued = 0; - int i; - int sched_prio; - bool will_fail = false; - unsigned long flags; - enum kbase_jd_atom_state status; - - dev_dbg(kbdev->dev, "User did JD submit atom %pK\n", (void *)katom); - - /* Update the TOTAL number of jobs. This includes those not tracked by - * the scheduler: 'not ready to run' and 'dependency-only' jobs. - */ - jctx->job_nr++; - -#if KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE - katom->start_timestamp.tv64 = 0; -#else - katom->start_timestamp = 0; -#endif - katom->udata = user_atom->udata; - katom->kctx = kctx; - katom->nr_extres = user_atom->nr_extres; - katom->extres = NULL; - katom->device_nr = user_atom->device_nr; - katom->jc = user_atom->jc; - katom->core_req = user_atom->core_req; - katom->jobslot = user_atom->jobslot; - katom->seq_nr = user_atom->seq_nr; - katom->atom_flags = 0; - katom->need_cache_flush_cores_retained = 0; - katom->pre_dep = NULL; - katom->post_dep = NULL; - katom->x_pre_dep = NULL; - katom->x_post_dep = NULL; - katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; - katom->softjob_data = NULL; - - trace_sysgraph(SGR_ARRIVE, kctx->id, user_atom->atom_number); - -#if MALI_JIT_PRESSURE_LIMIT_BASE - /* Older API version atoms might have random values where jit_id now - * lives, but we must maintain backwards compatibility - handle the - * issue. - */ - if (!mali_kbase_supports_jit_pressure_limit(kctx->api_version)) { - katom->jit_ids[0] = 0; - katom->jit_ids[1] = 0; - } else { - katom->jit_ids[0] = user_atom->jit_id[0]; - katom->jit_ids[1] = user_atom->jit_id[1]; - } -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /* Implicitly sets katom->protected_state.enter as well. */ - katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; - - katom->age = kctx->age_count++; - - INIT_LIST_HEAD(&katom->queue); - INIT_LIST_HEAD(&katom->jd_item); - - /* Don't do anything if there is a mess up with dependencies. - * This is done in a separate cycle to check both the dependencies at ones, otherwise - * it will be extra complexity to deal with 1st dependency ( just added to the list ) - * if only the 2nd one has invalid config. - */ - for (i = 0; i < 2; i++) { - int dep_atom_number = user_atom->pre_dep[i].atom_id; - base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type; - - if (dep_atom_number) { - if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER && - dep_atom_type != BASE_JD_DEP_TYPE_DATA) { - katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; - katom->status = KBASE_JD_ATOM_STATE_COMPLETED; - dev_dbg(kbdev->dev, "Atom %pK status to completed\n", - (void *)katom); - - /* Wrong dependency setup. Atom will be sent - * back to user space. Do not record any - * dependencies. - */ - jd_trace_atom_submit(kctx, katom, NULL); - return kbase_jd_done_nolock(katom, true); - } - } - } - - /* Add dependencies */ - for (i = 0; i < 2; i++) { - int dep_atom_number = user_atom->pre_dep[i].atom_id; - base_jd_dep_type dep_atom_type; - struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; - - dep_atom_type = user_atom->pre_dep[i].dependency_type; - kbase_jd_katom_dep_clear(&katom->dep[i]); - - if (!dep_atom_number) - continue; - - if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || - dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) { - if (dep_atom->event_code == BASE_JD_EVENT_DONE) - continue; - /* don't stop this atom if it has an order dependency - * only to the failed one, try to submit it through - * the normal path - */ - if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER && - dep_atom->event_code > BASE_JD_EVENT_ACTIVE) { - continue; - } - - /* Atom has completed, propagate the error code if any */ - katom->event_code = dep_atom->event_code; - katom->status = KBASE_JD_ATOM_STATE_QUEUED; - dev_dbg(kbdev->dev, "Atom %pK status to queued\n", (void *)katom); - - /* This atom will be sent back to user space. - * Do not record any dependencies. - */ - jd_trace_atom_submit(kctx, katom, NULL); - - will_fail = true; - - } else { - /* Atom is in progress, add this atom to the list */ - list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]); - kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type); - queued = 1; - } - } - - if (will_fail) { - if (!queued) { - if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { - /* This softjob has failed due to a previous - * dependency, however we should still run the - * prepare & finish functions - */ - int err = kbase_prepare_soft_job(katom); - - if (err >= 0) - kbase_finish_soft_job(katom); - } - return kbase_jd_done_nolock(katom, true); - } - - katom->will_fail_event_code = katom->event_code; - } - - /* These must occur after the above loop to ensure that an atom - * that depends on a previous atom with the same number behaves - * as expected - */ - katom->event_code = BASE_JD_EVENT_DONE; - katom->status = KBASE_JD_ATOM_STATE_QUEUED; - dev_dbg(kbdev->dev, "Atom %pK status to queued\n", (void *)katom); - - /* For invalid priority, be most lenient and choose the default */ - sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); - if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) - sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; - - /* Cap the priority to jctx.max_priority */ - katom->sched_priority = (sched_prio < kctx->jctx.max_priority) ? kctx->jctx.max_priority : - sched_prio; - - /* Create a new atom. */ - jd_trace_atom_submit(kctx, katom, &katom->sched_priority); - - if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { - /* Reject atoms with job chain = NULL, as these cause issues - * with soft-stop - */ - dev_err(kctx->kbdev->dev, "Rejecting atom with jc = NULL\n"); - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - - /* Reject atoms with an invalid device_nr */ - if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) && - (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) { - dev_err(kctx->kbdev->dev, "Rejecting atom with invalid device_nr %d\n", - katom->device_nr); - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - - /* Reject atoms with invalid core requirements */ - if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && - (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) { - dev_err(kctx->kbdev->dev, "Rejecting atom with invalid core requirements\n"); - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE; - return kbase_jd_done_nolock(katom, true); - } - - /* Reject soft-job atom of certain types from accessing external resources */ - if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && - (((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_FENCE_WAIT) || - ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_JIT_ALLOC) || - ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_JIT_FREE))) { - dev_err(kctx->kbdev->dev, "Rejecting soft-job atom accessing external resources\n"); - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - - if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { - /* handle what we need to do to access the external resources */ - if (kbase_jd_pre_external_resources(katom, user_atom) != 0) { - /* setup failed (no access, bad resource, unknown resource types, etc.) */ - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - } - -#if !MALI_JIT_PRESSURE_LIMIT_BASE - if (mali_kbase_supports_jit_pressure_limit(kctx->api_version) && - (user_atom->jit_id[0] || user_atom->jit_id[1])) { - /* JIT pressure limit is disabled, but we are receiving non-0 - * JIT IDs - atom is invalid. - */ - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /* Validate the atom. Function will return error if the atom is - * malformed. - * - * Soft-jobs never enter the job scheduler but have their own initialize method. - * - * If either fail then we immediately complete the atom with an error. - */ - if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) { - if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) { - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - } else { - /* Soft-job */ - if (kbase_prepare_soft_job(katom) != 0) { - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } - } - - if (queued && !IS_GPU_ATOM(katom)) - return false; - - if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { - if (kbase_process_soft_job(katom) == 0) { - kbase_finish_soft_job(katom); - return kbase_jd_done_nolock(katom, true); - } - return false; - } - - if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { - bool need_to_try_schedule_context; - - katom->status = KBASE_JD_ATOM_STATE_IN_JS; - dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n", (void *)katom); - - need_to_try_schedule_context = kbasep_js_add_job(kctx, katom); - /* If job was cancelled then resolve immediately */ - if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) - return need_to_try_schedule_context; - - /* Synchronize with backend reset */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - status = katom->status; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (status == KBASE_JD_ATOM_STATE_HW_COMPLETED) { - dev_dbg(kctx->kbdev->dev, "Atom %d cancelled on HW\n", - kbase_jd_atom_id(katom->kctx, katom)); - return need_to_try_schedule_context; - } - } - - /* This is a pure dependency. Resolve it immediately */ - return kbase_jd_done_nolock(katom, true); -} - -int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_atoms, u32 stride, - bool uk6_atom) -{ - struct kbase_jd_context *jctx = &kctx->jctx; - int err = 0; - u32 i; - bool need_to_try_schedule_context = false; - struct kbase_device *kbdev; - u32 latest_flush; - - bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2)); - - CSTD_UNUSED(uk6_atom); - - /* - * kbase_jd_submit isn't expected to fail and so all errors with the - * jobs are reported by immediately failing them (through event system) - */ - kbdev = kctx->kbdev; - - if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { - dev_err(kbdev->dev, - "Attempt to submit to a context that has SUBMIT_DISABLED set on it\n"); - return -EINVAL; - } - - if (stride != sizeof(struct base_jd_atom_v2) && stride != sizeof(struct base_jd_atom)) { - dev_err(kbdev->dev, - "Stride %u passed to job_submit isn't supported by the kernel\n", stride); - return -EINVAL; - } - - if (nr_atoms > BASE_JD_ATOM_COUNT) { - dev_dbg(kbdev->dev, "Invalid attempt to submit %u atoms at once for kctx %d_%d", - nr_atoms, kctx->tgid, kctx->id); - return -EINVAL; - } - - /* All atoms submitted in this call have the same flush ID */ - latest_flush = kbase_backend_get_current_flush_id(kbdev); - - for (i = 0; i < nr_atoms; i++) { - struct base_jd_atom user_atom = { - .seq_nr = 0, - }; - struct kbase_jd_atom *katom; - - if (unlikely(jd_atom_is_v2)) { - if (copy_from_user(&user_atom.jc, user_addr, - sizeof(struct base_jd_atom_v2)) != 0) { - dev_dbg(kbdev->dev, - "Invalid atom address %pK passed to job_submit\n", - user_addr); - err = -EFAULT; - break; - } - - /* no seq_nr in v2 */ - user_atom.seq_nr = 0; - } else { - if (copy_from_user(&user_atom, user_addr, stride) != 0) { - dev_dbg(kbdev->dev, - "Invalid atom address %pK passed to job_submit\n", - user_addr); - err = -EFAULT; - break; - } - } - - user_addr = (void __user *)((uintptr_t)user_addr + stride); - - mutex_lock(&jctx->lock); -#ifndef compiletime_assert -#define compiletime_assert_defined -#define compiletime_assert(x, msg) \ - do { \ - switch (0) { \ - case 0: \ - case (x):; \ - } \ - } while (false) -#endif - compiletime_assert((1 << (8 * sizeof(user_atom.atom_number))) == BASE_JD_ATOM_COUNT, - "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); - compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) == - sizeof(user_atom.atom_number), - "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); -#ifdef compiletime_assert_defined -#undef compiletime_assert -#undef compiletime_assert_defined -#endif - katom = &jctx->atoms[user_atom.atom_number]; - - /* Record the flush ID for the cache flush optimisation */ - katom->flush_id = latest_flush; - - while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) { - /* Atom number is already in use, wait for the atom to - * complete - */ - mutex_unlock(&jctx->lock); - - /* This thread will wait for the atom to complete. Due - * to thread scheduling we are not sure that the other - * thread that owns the atom will also schedule the - * context, so we force the scheduler to be active and - * hence eventually schedule this context at some point - * later. - */ - kbase_js_sched_all(kbdev); - - if (wait_event_killable(katom->completed, - katom->status == KBASE_JD_ATOM_STATE_UNUSED) != 0) { - /* We're being killed so the result code - * doesn't really matter - */ - return 0; - } - mutex_lock(&jctx->lock); - } - KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_START(kbdev, katom); - need_to_try_schedule_context |= jd_submit_atom(kctx, &user_atom, katom); - KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_END(kbdev, katom); - /* Register a completed job as a disjoint event when the GPU is in a disjoint state - * (ie. being reset). - */ - kbase_disjoint_event_potential(kbdev); - - mutex_unlock(&jctx->lock); - if (fatal_signal_pending(current)) { - dev_dbg(kbdev->dev, "Fatal signal pending for kctx %d_%d", kctx->tgid, - kctx->id); - /* We're being killed so the result code doesn't really matter */ - return 0; - } - } - - if (need_to_try_schedule_context) - kbase_js_sched_all(kbdev); - - return err; -} - -KBASE_EXPORT_TEST_API(kbase_jd_submit); - -void kbase_jd_done_worker(struct work_struct *data) -{ - struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); - struct kbase_jd_context *jctx; - struct kbase_context *kctx; - struct kbasep_js_kctx_info *js_kctx_info; - struct kbase_device *kbdev; - struct kbasep_js_device_data *js_devdata; - u64 cache_jc = katom->jc; - struct kbasep_js_atom_retained_state katom_retained_state; - bool context_idle; - base_jd_core_req core_req = katom->core_req; - - /* Soft jobs should never reach this function */ - KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); - - kctx = katom->kctx; - jctx = &kctx->jctx; - kbdev = kctx->kbdev; - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - - dev_dbg(kbdev->dev, "Enter atom %pK done worker for kctx %pK\n", (void *)katom, - (void *)kctx); - - KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); - - kbase_backend_complete_wq(kbdev, katom); - - /* - * Begin transaction on JD context and JS context - */ - mutex_lock(&jctx->lock); - KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_DONE); - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - - /* This worker only gets called on contexts that are scheduled *in*. This is - * because it only happens in response to an IRQ from a job that was - * running. - */ - KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - if (katom->event_code == BASE_JD_EVENT_STOPPED) { - unsigned long flags; - - dev_dbg(kbdev->dev, "Atom %pK has been promoted to stopped\n", (void *)katom); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - katom->status = KBASE_JD_ATOM_STATE_IN_JS; - dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n", (void *)katom); - kbase_js_unpull(kctx, katom); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&jctx->lock); - - return; - } - - if ((katom->event_code != BASE_JD_EVENT_DONE) && !kbase_ctx_flag(katom->kctx, KCTX_DYING) && - !kbase_ctx_flag(katom->kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) - if (!kbase_is_quick_reset_enabled(kbdev)) - dev_err(kbdev->dev, "t6xx: GPU fault 0x%02lx from job slot %d\n", - (unsigned long)katom->event_code, katom->slot_nr); - - /* Retain state before the katom disappears */ - kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); - - context_idle = kbase_js_complete_atom_wq(kctx, katom); - - KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state)); - - kbasep_js_remove_job(kbdev, kctx, katom); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - /* kbase_jd_done_nolock() requires the jsctx_mutex lock to be dropped */ - kbase_jd_done_nolock(katom, false); - - /* katom may have been freed now, do not use! */ - - if (context_idle) { - unsigned long flags; - - context_idle = false; - mutex_lock(&js_devdata->queue_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* If kbase_sched() has scheduled this context back in then - * KCTX_ACTIVE will have been set after we marked it as - * inactive, and another pm reference will have been taken, so - * drop our reference. But do not call kbase_jm_idle_ctx(), as - * the context is active and fast-starting is allowed. - * - * If an atom has been fast-started then - * kbase_jsctx_atoms_pulled(kctx) will return non-zero but - * KCTX_ACTIVE will still be false (as the previous pm - * reference has been inherited). Do NOT drop our reference, as - * it has been re-used, and leave the context as active. - * - * If no new atoms have been started then KCTX_ACTIVE will - * still be false and kbase_jsctx_atoms_pulled(kctx) will - * return zero, so drop the reference and call - * kbase_jm_idle_ctx(). - * - * As the checks are done under both the queue_mutex and - * hwaccess_lock is should be impossible for this to race - * with the scheduler code. - */ - if (kbase_ctx_flag(kctx, KCTX_ACTIVE) || !kbase_jsctx_atoms_pulled(kctx)) { - /* Calling kbase_jm_idle_ctx() here will ensure that - * atoms are not fast-started when we drop the - * hwaccess_lock. This is not performed if - * KCTX_ACTIVE is set as in that case another pm - * reference has been taken and a fast-start would be - * valid. - */ - if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) - kbase_jm_idle_ctx(kbdev, kctx); - context_idle = true; - } else { - kbase_ctx_flag_set(kctx, KCTX_ACTIVE); - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_devdata->queue_mutex); - } - - /* - * Transaction complete - */ - mutex_unlock(&jctx->lock); - - /* Job is now no longer running, so can now safely release the context - * reference, and handle any actions that were logged against the - * atom's retained state - */ - - kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); - - kbase_js_sched_all(kbdev); - - if (!atomic_dec_return(&kctx->work_count)) { - /* If worker now idle then post all events that kbase_jd_done_nolock() - * has queued - */ - mutex_lock(&jctx->lock); - while (!list_empty(&kctx->completed_jobs)) { - struct kbase_jd_atom *atom = list_entry(kctx->completed_jobs.next, - struct kbase_jd_atom, jd_item); - list_del(kctx->completed_jobs.next); - - kbase_event_post(kctx, atom); - } - mutex_unlock(&jctx->lock); - } - - kbase_backend_complete_wq_post_sched(kbdev, core_req); - - if (context_idle) - kbase_pm_context_idle(kbdev); - - KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); - - dev_dbg(kbdev->dev, "Leave atom %pK done worker for kctx %pK\n", (void *)katom, - (void *)kctx); -} - -/** - * jd_cancel_worker - Work queue job cancel function. - * @data: a &struct work_struct - * - * Only called as part of 'Zapping' a context (which occurs on termination). - * Operates serially with the kbase_jd_done_worker() on the work queue. - * - * This can only be called on contexts that aren't scheduled. - * - * We don't need to release most of the resources that would occur on - * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be - * running (by virtue of only being called on contexts that aren't - * scheduled). - */ -static void jd_cancel_worker(struct work_struct *data) -{ - struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); - struct kbase_jd_context *jctx; - struct kbase_context *kctx; - struct kbasep_js_kctx_info *js_kctx_info; - bool need_to_try_schedule_context; - bool attr_state_changed; - struct kbase_device *kbdev; - CSTD_UNUSED(need_to_try_schedule_context); - - /* Soft jobs should never reach this function */ - KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); - - kctx = katom->kctx; - kbdev = kctx->kbdev; - jctx = &kctx->jctx; - js_kctx_info = &kctx->jctx.sched_info; - - KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); - - /* This only gets called on contexts that are scheduled out. Hence, we must - * make sure we don't de-ref the number of running jobs (there aren't - * any), nor must we try to schedule out the context (it's already - * scheduled out). - */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - /* Scheduler: Remove the job from the system */ - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - mutex_lock(&jctx->lock); - - need_to_try_schedule_context = kbase_jd_done_nolock(katom, true); - /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to - * schedule the context. There's also no need for the jsctx_mutex to have been taken - * around this too. - */ - KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); - - /* katom may have been freed now, do not use! */ - mutex_unlock(&jctx->lock); - - if (attr_state_changed) - kbase_js_sched_all(kbdev); -} - -/** - * kbase_jd_done - Complete a job that has been removed from the Hardware - * @katom: atom which has been completed - * @slot_nr: slot the atom was on - * @end_timestamp: completion time - * @done_code: completion code - * - * This must be used whenever a job has been removed from the Hardware, e.g.: - * An IRQ indicates that the job finished (for both error and 'done' codes), or - * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop. - * - * Some work is carried out immediately, and the rest is deferred onto a - * workqueue - * - * Context: - * This can be called safely from atomic context. - * The caller must hold kbdev->hwaccess_lock - */ -void kbase_jd_done(struct kbase_jd_atom *katom, unsigned int slot_nr, ktime_t *end_timestamp, - kbasep_js_atom_done_code done_code) -{ - struct kbase_context *kctx; - struct kbase_device *kbdev; - - CSTD_UNUSED(end_timestamp); - - KBASE_DEBUG_ASSERT(katom); - kctx = katom->kctx; - KBASE_DEBUG_ASSERT(kctx); - kbdev = kctx->kbdev; - KBASE_DEBUG_ASSERT(kbdev); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) - katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; - - KBASE_KTRACE_ADD_JM(kbdev, JD_DONE, kctx, katom, katom->jc, 0); - - kbase_job_check_leave_disjoint(kbdev, katom); - - katom->slot_nr = slot_nr; - - atomic_inc(&kctx->work_count); - -#if IS_ENABLED(CONFIG_DEBUG_FS) - /* a failed job happened and is waiting for dumping*/ - if (!katom->will_fail_event_code && kbase_debug_job_fault_process(katom, katom->event_code)) - return; -#endif - - WARN_ON(work_pending(&katom->work)); - INIT_WORK(&katom->work, kbase_jd_done_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); -} - -KBASE_EXPORT_TEST_API(kbase_jd_done); - -void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(katom != NULL); - kctx = katom->kctx; - KBASE_DEBUG_ASSERT(kctx != NULL); - - dev_dbg(kbdev->dev, "JD: cancelling atom %pK\n", (void *)katom); - KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); - - /* This should only be done from a context that is not scheduled */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - WARN_ON(work_pending(&katom->work)); - - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - INIT_WORK(&katom->work, jd_cancel_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); -} - -void kbase_jd_zap_context(struct kbase_context *kctx) -{ - struct kbase_jd_atom *katom; - struct list_head *entry, *tmp; - struct kbase_device *kbdev; - - KBASE_DEBUG_ASSERT(kctx); - - kbdev = kctx->kbdev; - - KBASE_KTRACE_ADD_JM(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); - - kbase_js_zap_context(kctx); - - mutex_lock(&kctx->jctx.lock); - - /* - * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are - * queued outside the job scheduler. - */ - - del_timer_sync(&kctx->soft_job_timeout); - list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { - katom = list_entry(entry, struct kbase_jd_atom, queue); - kbase_cancel_soft_job(katom); - } - - mutex_unlock(&kctx->jctx.lock); - -#if IS_ENABLED(CONFIG_DEBUG_FS) - kbase_debug_job_fault_kctx_unblock(kctx); -#endif - - kbase_jm_wait_for_zero_jobs(kctx); -} - -KBASE_EXPORT_TEST_API(kbase_jd_zap_context); - -int kbase_jd_init(struct kbase_context *kctx) -{ - int i; - int mali_err = 0; - struct priority_control_manager_device *pcm_device = NULL; - - KBASE_DEBUG_ASSERT(kctx); - pcm_device = kctx->kbdev->pcm_dev; - kctx->jctx.max_priority = KBASE_JS_ATOM_SCHED_PRIO_REALTIME; - - kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", WQ_HIGHPRI | WQ_UNBOUND, 1); - if (kctx->jctx.job_done_wq == NULL) { - mali_err = -ENOMEM; - goto out1; - } - - for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { - init_waitqueue_head(&kctx->jctx.atoms[i].completed); - - INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]); - INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]); - - /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */ - kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; - kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; - -#if IS_ENABLED(CONFIG_SYNC_FILE) - kctx->jctx.atoms[i].dma_fence.context = dma_fence_context_alloc(1); -#endif - } - - mutex_init(&kctx->jctx.lock); - - init_waitqueue_head(&kctx->jctx.zero_jobs_wait); - - spin_lock_init(&kctx->jctx.tb_lock); - - INIT_LIST_HEAD(&kctx->completed_jobs); - - /* Check if there are platform rules for maximum priority */ - if (pcm_device) - kctx->jctx.max_priority = pcm_device->ops.pcm_scheduler_priority_check( - pcm_device, current, KBASE_JS_ATOM_SCHED_PRIO_REALTIME); - - return 0; - -out1: - return mali_err; -} - -KBASE_EXPORT_TEST_API(kbase_jd_init); - -void kbase_jd_exit(struct kbase_context *kctx) -{ - KBASE_DEBUG_ASSERT(kctx); - - /* Work queue is emptied by this */ - destroy_workqueue(kctx->jctx.job_done_wq); -} - -KBASE_EXPORT_TEST_API(kbase_jd_exit); diff --git a/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.c b/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.c deleted file mode 100644 index c3b6a63afc3f..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.c +++ /dev/null @@ -1,189 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#if IS_ENABLED(CONFIG_DEBUG_FS) - -#include -#include -#include -#if IS_ENABLED(CONFIG_SYNC_FILE) -#include -#endif -#include - -struct kbase_jd_debugfs_depinfo { - u8 id; - char type; -}; - -static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, struct seq_file *sfile) -{ -#if IS_ENABLED(CONFIG_SYNC_FILE) - struct kbase_sync_fence_info info; - int res; - - switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_FENCE_TRIGGER: - res = kbase_sync_fence_out_info_get(atom, &info); - if (res == 0) - seq_printf(sfile, "Sa([%pK]%d) ", info.fence, info.status); - break; - case BASE_JD_REQ_SOFT_FENCE_WAIT: - res = kbase_sync_fence_in_info_get(atom, &info); - if (res == 0) - seq_printf(sfile, "Wa([%pK]%d) ", info.fence, info.status); - break; - default: - break; - } -#endif /* CONFIG_SYNC_FILE */ -} - -static void kbasep_jd_debugfs_atom_deps(struct kbase_jd_debugfs_depinfo *deps, - struct kbase_jd_atom *atom) -{ - struct kbase_context *kctx = atom->kctx; - int i; - - for (i = 0; i < 2; i++) { - deps[i].id = (unsigned int)(atom->dep[i].atom ? - kbase_jd_atom_id(kctx, atom->dep[i].atom) : - 0); - - switch (atom->dep[i].dep_type) { - case BASE_JD_DEP_TYPE_INVALID: - deps[i].type = ' '; - break; - case BASE_JD_DEP_TYPE_DATA: - deps[i].type = 'D'; - break; - case BASE_JD_DEP_TYPE_ORDER: - deps[i].type = '>'; - break; - default: - deps[i].type = '?'; - break; - } - } -} -/** - * kbasep_jd_debugfs_atoms_show - Show callback for the JD atoms debugfs file. - * @sfile: The debugfs entry - * @data: Data associated with the entry - * - * This function is called to get the contents of the JD atoms debugfs file. - * This is a report of all atoms managed by kbase_jd_context.atoms - * - * Return: 0 if successfully prints data in debugfs entry file, failure - * otherwise - */ -static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) -{ - struct kbase_context *kctx = sfile->private; - struct kbase_jd_atom *atoms; - unsigned long irq_flags; - int i; - - CSTD_UNUSED(data); - - KBASE_DEBUG_ASSERT(kctx != NULL); - - /* Print version */ - seq_printf(sfile, "v%u\n", MALI_JD_DEBUGFS_VERSION); - - /* Print U/K API version */ - seq_printf(sfile, "ukv%u.%u\n", BASE_UK_VERSION_MAJOR, BASE_UK_VERSION_MINOR); - - /* Print table heading */ - seq_puts(sfile, " ID, Core req, St, Predeps, Start time, Additional info...\n"); - - atoms = kctx->jctx.atoms; - /* General atom states */ - mutex_lock(&kctx->jctx.lock); - /* JS-related states */ - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); - for (i = 0; i != BASE_JD_ATOM_COUNT; ++i) { - struct kbase_jd_atom *atom = &atoms[i]; - s64 start_timestamp = 0; - struct kbase_jd_debugfs_depinfo deps[2]; - - if (atom->status == KBASE_JD_ATOM_STATE_UNUSED) - continue; - - /* start_timestamp is cleared as soon as the atom leaves UNUSED state - * and set before a job is submitted to the h/w, a non-zero value means - * it is valid - */ - if (ktime_to_ns(atom->start_timestamp)) - start_timestamp = - ktime_to_ns(ktime_sub(ktime_get_raw(), atom->start_timestamp)); - - kbasep_jd_debugfs_atom_deps(deps, atom); - - seq_printf(sfile, "%3u, %8x, %2u, %c%3u %c%3u, %20lld, ", i, atom->core_req, - atom->status, deps[0].type, deps[0].id, deps[1].type, deps[1].id, - start_timestamp); - - kbase_jd_debugfs_fence_info(atom, sfile); - - seq_puts(sfile, "\n"); - } - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); - mutex_unlock(&kctx->jctx.lock); - - return 0; -} - -/** - * kbasep_jd_debugfs_atoms_open - open operation for atom debugfs file - * @in: &struct inode pointer - * @file: &struct file pointer - * - * Return: file descriptor - */ -static int kbasep_jd_debugfs_atoms_open(struct inode *in, struct file *file) -{ - return single_open(file, kbasep_jd_debugfs_atoms_show, in->i_private); -} - -static const struct file_operations kbasep_jd_debugfs_atoms_fops = { - .owner = THIS_MODULE, - .open = kbasep_jd_debugfs_atoms_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) -{ - const mode_t mode = 0444; - - /* Caller already ensures this, but we keep the pattern for - * maintenance safety. - */ - if (WARN_ON(!kctx) || WARN_ON(IS_ERR_OR_NULL(kctx->kctx_dentry))) - return; - - /* Expose all atoms */ - debugfs_create_file("atoms", mode, kctx->kctx_dentry, kctx, &kbasep_jd_debugfs_atoms_fops); -} - -#endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.h b/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.h deleted file mode 100644 index 8ea18b36d0d8..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_jd_debugfs.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Header file for job dispatcher-related entries in debugfs - */ - -#ifndef _KBASE_JD_DEBUGFS_H -#define _KBASE_JD_DEBUGFS_H - -#include - -#define MALI_JD_DEBUGFS_VERSION 3 - -/* Forward declarations */ -struct kbase_context; - -/** - * kbasep_jd_debugfs_ctx_init() - Add debugfs entries for JD system - * - * @kctx: Pointer to kbase_context - */ -void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx); - -#endif /*_KBASE_JD_DEBUGFS_H*/ diff --git a/drivers/gpu/arm/valhall/mali_kbase_jm.c b/drivers/gpu/arm/valhall/mali_kbase_jm.c deleted file mode 100644 index cac12df225a0..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_jm.c +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * HW access job manager common APIs - */ - -#include -#include "mali_kbase_hwaccess_jm.h" -#include "mali_kbase_jm.h" - -#if !MALI_USE_CSF -/** - * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot - * @js on the active context. - * @kbdev: Device pointer - * @js: Job slot to run on - * @nr_jobs_to_submit: Number of jobs to attempt to submit - * - * Return: true if slot can still be submitted on, false if slot is now full. - */ -static bool kbase_jm_next_job(struct kbase_device *kbdev, unsigned int js, int nr_jobs_to_submit) -{ - struct kbase_context *kctx; - int i; - - kctx = kbdev->hwaccess.active_kctx[js]; - dev_dbg(kbdev->dev, "Trying to run the next %d jobs in kctx %pK (s:%u)\n", - nr_jobs_to_submit, (void *)kctx, js); - - if (!kctx) - return true; - - for (i = 0; i < nr_jobs_to_submit; i++) { - struct kbase_jd_atom *katom = kbase_js_pull(kctx, js); - - if (!katom) - return true; /* Context has no jobs on this slot */ - - kbase_backend_run_atom(kbdev, katom); - } - - dev_dbg(kbdev->dev, "Slot ringbuffer should now be full (s:%u)\n", js); - return false; -} - -u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask) -{ - u32 ret_mask = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "JM kick slot mask 0x%x\n", js_mask); - - while (js_mask) { - unsigned int js = (unsigned int)ffs((int)js_mask) - 1U; - int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js); - - if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit)) - ret_mask |= (1U << js); - - js_mask &= ~(1U << js); - } - - dev_dbg(kbdev->dev, "Can still submit to mask 0x%x\n", ret_mask); - return ret_mask; -} - -void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask) -{ - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (!down_trylock(&js_devdata->schedule_sem)) { - kbase_jm_kick(kbdev, js_mask); - up(&js_devdata->schedule_sem); - } -} - -void kbase_jm_try_kick_all(struct kbase_device *kbdev) -{ - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (!down_trylock(&js_devdata->schedule_sem)) { - kbase_jm_kick_all(kbdev); - up(&js_devdata->schedule_sem); - } -} - -void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - unsigned int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - if (kbdev->hwaccess.active_kctx[js] == kctx) { - dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%u)\n", (void *)kctx, - js); - kbdev->hwaccess.active_kctx[js] = NULL; - } - } -} - -struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, - struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - dev_dbg(kbdev->dev, "Atom %pK is returning with event code 0x%x\n", (void *)katom, - katom->event_code); - - KBASE_KTRACE_ADD_JM(kbdev, JM_RETURN_ATOM_TO_JS, katom->kctx, katom, katom->jc, - katom->event_code); - - if (katom->event_code != BASE_JD_EVENT_STOPPED && - katom->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT) { - return kbase_js_complete_atom(katom, NULL); - } - - kbase_js_unpull(katom->kctx, katom); - - return NULL; -} - -struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - ktime_t *end_timestamp) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - return kbase_js_complete_atom(katom, end_timestamp); -} -#endif /* !MALI_USE_CSF */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_jm.h b/drivers/gpu/arm/valhall/mali_kbase_jm.h deleted file mode 100644 index fb0e69a0aee3..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_jm.h +++ /dev/null @@ -1,117 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Job manager common APIs - */ - -#ifndef _KBASE_JM_H_ -#define _KBASE_JM_H_ - -#if !MALI_USE_CSF -/** - * kbase_jm_kick() - Indicate that there are jobs ready to run. - * @kbdev: Device pointer - * @js_mask: Mask of the job slots that can be pulled from. - * - * Caller must hold the hwaccess_lock and schedule_sem semaphore - * - * Return: Mask of the job slots that can still be submitted to. - */ -u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask); - -/** - * kbase_jm_kick_all() - Indicate that there are jobs ready to run on all job - * slots. - * @kbdev: Device pointer - * - * Caller must hold the hwaccess_lock and schedule_sem semaphore - * - * Return: Mask of the job slots that can still be submitted to. - */ -static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev) -{ - return kbase_jm_kick(kbdev, (1U << kbdev->gpu_props.num_job_slots) - 1U); -} - -/** - * kbase_jm_try_kick - Attempt to call kbase_jm_kick - * @kbdev: Device pointer - * @js_mask: Mask of the job slots that can be pulled from - * Context: Caller must hold hwaccess_lock - * - * If schedule_sem can be immediately obtained then this function will call - * kbase_jm_kick() otherwise it will do nothing. - */ -void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask); - -/** - * kbase_jm_try_kick_all() - Attempt to call kbase_jm_kick_all - * @kbdev: Device pointer - * Context: Caller must hold hwaccess_lock - * - * If schedule_sem can be immediately obtained then this function will call - * kbase_jm_kick_all() otherwise it will do nothing. - */ -void kbase_jm_try_kick_all(struct kbase_device *kbdev); -#endif /* !MALI_USE_CSF */ - -#if !MALI_USE_CSF -/** - * kbase_jm_idle_ctx() - Mark a context as idle. - * @kbdev: Device pointer - * @kctx: Context to mark as idle - * - * No more atoms will be pulled from this context until it is marked as active - * by kbase_js_use_ctx(). - * - * The context should have no atoms currently pulled from it - * (kbase_jsctx_atoms_pulled(kctx) == 0). - * - * Caller must hold the hwaccess_lock - */ -void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbase_jm_return_atom_to_js() - Return an atom to the job scheduler that has - * been soft-stopped or will fail due to a - * dependency - * @kbdev: Device pointer - * @katom: Atom that has been stopped or will be failed - * - * Return: Atom that has now been unblocked and can now be run, or NULL if none - */ -struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, - struct kbase_jd_atom *katom); - -/** - * kbase_jm_complete() - Complete an atom - * @kbdev: Device pointer - * @katom: Atom that has completed - * @end_timestamp: Timestamp of atom completion - * - * Return: Atom that has now been unblocked and can now be run, or NULL if none - */ -struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, struct kbase_jd_atom *katom, - ktime_t *end_timestamp); -#endif /* !MALI_USE_CSF */ - -#endif /* _KBASE_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_js.c b/drivers/gpu/arm/valhall/mali_kbase_js.c deleted file mode 100644 index 8e3dd7023373..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_js.c +++ /dev/null @@ -1,3271 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * Job Scheduler Implementation - */ -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "mali_kbase_jm.h" -#include "mali_kbase_hwaccess_jm.h" -#include -#include -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) -#include -#endif - -/* - * Private types - */ - -/* Bitpattern indicating the result of releasing a context */ -enum { - /* The context was descheduled - caller should try scheduling in a new - * one to keep the runpool full - */ - KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), - /* Ctx attributes were changed - caller should try scheduling all - * contexts - */ - KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) -}; - -typedef u32 kbasep_js_release_result; - -const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { - KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ - KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ - KBASE_JS_ATOM_SCHED_PRIO_LOW, /* BASE_JD_PRIO_LOW */ - KBASE_JS_ATOM_SCHED_PRIO_REALTIME /* BASE_JD_PRIO_REALTIME */ -}; - -const base_jd_prio kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { - BASE_JD_PRIO_REALTIME, /* KBASE_JS_ATOM_SCHED_PRIO_REALTIME */ - BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ - BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ - BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ -}; - -/* - * Private function prototypes - */ -static kbasep_js_release_result -kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state); - -static unsigned int kbase_js_get_slot(struct kbase_device *kbdev, struct kbase_jd_atom *katom); - -static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, kbasep_js_ctx_job_cb *callback); - -/* Helper for ktrace */ -#if KBASE_KTRACE_ENABLE -static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) -{ - return atomic_read(&kctx->refcount); -} -#else /* KBASE_KTRACE_ENABLE */ -static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) -{ - CSTD_UNUSED(kctx); - return 0; -} -#endif /* KBASE_KTRACE_ENABLE */ - -/* - * Private functions - */ - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) -/** - * gpu_metrics_timer_callback() - Callback function for the GPU metrics hrtimer - * - * @timer: Pointer to the GPU metrics hrtimer - * - * This function will emit power/gpu_work_period tracepoint for all the active - * GPU metrics contexts. The timer will be restarted if needed. - * - * Return: enum value to indicate that timer should not be restarted. - */ -static enum hrtimer_restart gpu_metrics_timer_callback(struct hrtimer *timer) -{ - struct kbasep_js_device_data *js_devdata = - container_of(timer, struct kbasep_js_device_data, gpu_metrics_timer); - struct kbase_device *kbdev = container_of(js_devdata, struct kbase_device, js_data); - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_gpu_metrics_emit_tracepoint(kbdev, ktime_get_raw_ns()); - WARN_ON_ONCE(!js_devdata->gpu_metrics_timer_running); - if (js_devdata->gpu_metrics_timer_needed) { - hrtimer_start(&js_devdata->gpu_metrics_timer, - HR_TIMER_DELAY_NSEC(kbase_gpu_metrics_get_tp_emit_interval()), - HRTIMER_MODE_REL); - } else - js_devdata->gpu_metrics_timer_running = false; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return HRTIMER_NORESTART; -} - -/** - * gpu_metrics_ctx_init() - Take a reference on GPU metrics context if it exists, - * otherwise allocate and initialise one. - * - * @kctx: Pointer to the Kbase context. - * - * The GPU metrics context represents an "Application" for the purposes of GPU metrics - * reporting. There may be multiple kbase_contexts contributing data to a single GPU - * metrics context. - * This function takes a reference on GPU metrics context if it already exists - * corresponding to the Application that is creating the Kbase context, otherwise - * memory is allocated for it and initialised. - * - * Return: 0 on success, or negative on failure. - */ -static inline int gpu_metrics_ctx_init(struct kbase_context *kctx) -{ - struct kbase_gpu_metrics_ctx *gpu_metrics_ctx; - struct kbase_device *kbdev = kctx->kbdev; - unsigned long flags; - int ret = 0; - - const struct cred *cred = get_current_cred(); - const unsigned int aid = cred->euid.val; - - put_cred(cred); - - /* Return early if this is not a Userspace created context */ - if (unlikely(!kctx->filp)) - return 0; - - /* Serialize against the other threads trying to create/destroy Kbase contexts. */ - mutex_lock(&kbdev->kctx_list_lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - gpu_metrics_ctx = kbase_gpu_metrics_ctx_get(kbdev, aid); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (!gpu_metrics_ctx) { - gpu_metrics_ctx = kmalloc(sizeof(*gpu_metrics_ctx), GFP_KERNEL); - - if (gpu_metrics_ctx) { - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_gpu_metrics_ctx_init(kbdev, gpu_metrics_ctx, aid); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } else { - dev_err(kbdev->dev, "Allocation for gpu_metrics_ctx failed"); - ret = -ENOMEM; - } - } - - kctx->gpu_metrics_ctx = gpu_metrics_ctx; - mutex_unlock(&kbdev->kctx_list_lock); - - return ret; -} - -/** - * gpu_metrics_ctx_term() - Drop a reference on a GPU metrics context and free it - * if the refcount becomes 0. - * - * @kctx: Pointer to the Kbase context. - */ -static inline void gpu_metrics_ctx_term(struct kbase_context *kctx) -{ - unsigned long flags; - - /* Return early if this is not a Userspace created context */ - if (unlikely(!kctx->filp)) - return; - - /* Serialize against the other threads trying to create/destroy Kbase contexts. */ - mutex_lock(&kctx->kbdev->kctx_list_lock); - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); - kbase_gpu_metrics_ctx_put(kctx->kbdev, kctx->gpu_metrics_ctx); - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); - mutex_unlock(&kctx->kbdev->kctx_list_lock); -} -#endif - -/** - * core_reqs_from_jsn_features - Convert JSn_FEATURES to core requirements - * @features: parsed JSn_FEATURE register value - * - * Given a JSn_FEATURE register value returns the core requirements that match - * - * Return: Core requirement bit mask - */ -static base_jd_core_req core_reqs_from_jsn_features(struct kbase_js_features_props *features) -{ - base_jd_core_req core_req = 0u; - - if (features->write_value) - core_req |= BASE_JD_REQ_V; - - if (features->cache_flush) - core_req |= BASE_JD_REQ_CF; - - if (features->compute_shader) - core_req |= BASE_JD_REQ_CS; - - if (features->tiler) - core_req |= BASE_JD_REQ_T; - - if (features->fragment_shader) - core_req |= BASE_JD_REQ_FS; - - return core_req; -} - -static void kbase_js_sync_timers(struct kbase_device *kbdev) -{ - mutex_lock(&kbdev->js_data.runpool_mutex); - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&kbdev->js_data.runpool_mutex); -} - -/** - * jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms - * @kctx: Pointer to kbase context with ring buffer. - * @js: Job slot id to check. - * @prio: Priority to check. - * - * Return true if there are no atoms to pull. There may be running atoms in the - * ring buffer even if there are no atoms to pull. It is also possible for the - * ring buffer to be full (with running atoms) when this functions returns - * true. - * - * Return: true if there are no atoms to pull, false otherwise. - */ -static inline bool jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, unsigned int js, int prio) -{ - bool none_to_pull; - struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - none_to_pull = RB_EMPTY_ROOT(&rb->runnable_tree); - - dev_dbg(kctx->kbdev->dev, "Slot %u (prio %d) is %spullable in kctx %pK\n", js, prio, - none_to_pull ? "not " : "", kctx); - - return none_to_pull; -} - -/** - * jsctx_rb_none_to_pull(): - Check if all priority ring buffers have no - * pullable atoms - * @kctx: Pointer to kbase context with ring buffer. - * @js: Job slot id to check. - * - * Caller must hold hwaccess_lock - * - * Return: true if the ring buffers for all priorities have no pullable atoms, - * false otherwise. - */ -static inline bool jsctx_rb_none_to_pull(struct kbase_context *kctx, unsigned int js) -{ - int prio; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { - if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) - return false; - } - - return true; -} - -/** - * jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue. - * @kctx: Pointer to kbase context with the queue. - * @js: Job slot id to iterate. - * @prio: Priority id to iterate. - * @callback: Function pointer to callback. - * - * Iterate over a queue and invoke @callback for each entry in the queue, and - * remove the entry from the queue. - * - * If entries are added to the queue while this is running those entries may, or - * may not be covered. To ensure that all entries in the buffer have been - * enumerated when this function returns jsctx->lock must be held when calling - * this function. - * - * The HW access lock must always be held when calling this function. - */ -static void jsctx_queue_foreach_prio(struct kbase_context *kctx, unsigned int js, int prio, - kbasep_js_ctx_job_cb *callback) -{ - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - while (!RB_EMPTY_ROOT(&queue->runnable_tree)) { - struct rb_node *node = rb_first(&queue->runnable_tree); - struct kbase_jd_atom *entry = - rb_entry(node, struct kbase_jd_atom, runnable_tree_node); - - rb_erase(node, &queue->runnable_tree); - callback(kctx->kbdev, entry); - } - - while (!list_empty(&queue->x_dep_head)) { - struct kbase_jd_atom *entry = - list_entry(queue->x_dep_head.next, struct kbase_jd_atom, queue); - - WARN_ON(!(entry->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); - dev_dbg(kctx->kbdev->dev, "Del blocked atom %pK from X_DEP list\n", (void *)entry); - - list_del(queue->x_dep_head.next); - entry->atom_flags &= ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - - callback(kctx->kbdev, entry); - } -} - -/** - * jsctx_queue_foreach(): - Execute callback for each entry in every queue - * @kctx: Pointer to kbase context with queue. - * @js: Job slot id to iterate. - * @callback: Function pointer to callback. - * - * Iterate over all the different priorities, and for each call - * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback - * for each entry, and remove the entry from the queue. - */ -static inline void jsctx_queue_foreach(struct kbase_context *kctx, unsigned int js, - kbasep_js_ctx_job_cb *callback) -{ - int prio; - - for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) - jsctx_queue_foreach_prio(kctx, js, prio, callback); -} - -/** - * jsctx_rb_peek_prio(): - Check buffer and get next atom - * @kctx: Pointer to kbase context with ring buffer. - * @js: Job slot id to check. - * @prio: Priority id to check. - * - * Check the ring buffer for the specified @js and @prio and return a pointer to - * the next atom, unless the ring buffer is empty. - * - * Return: Pointer to next atom in buffer, or NULL if there is no atom. - */ -static inline struct kbase_jd_atom *jsctx_rb_peek_prio(struct kbase_context *kctx, unsigned int js, - int prio) -{ - struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; - struct rb_node *node; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - dev_dbg(kctx->kbdev->dev, "Peeking runnable tree of kctx %pK for prio %d (s:%u)\n", - (void *)kctx, prio, js); - - node = rb_first(&rb->runnable_tree); - if (!node) { - dev_dbg(kctx->kbdev->dev, "Tree is empty\n"); - return NULL; - } - - return rb_entry(node, struct kbase_jd_atom, runnable_tree_node); -} - -/** - * jsctx_rb_peek(): - Check all priority buffers and get next atom - * @kctx: Pointer to kbase context with ring buffer. - * @js: Job slot id to check. - * - * Check the ring buffers for all priorities, starting from - * KBASE_JS_ATOM_SCHED_PRIO_REALTIME, for the specified @js and @prio and return a - * pointer to the next atom, unless all the priority's ring buffers are empty. - * - * Caller must hold the hwaccess_lock. - * - * Return: Pointer to next atom in buffer, or NULL if there is no atom. - */ -static inline struct kbase_jd_atom *jsctx_rb_peek(struct kbase_context *kctx, unsigned int js) -{ - int prio; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { - struct kbase_jd_atom *katom; - - katom = jsctx_rb_peek_prio(kctx, js, prio); - if (katom) - return katom; - } - - return NULL; -} - -/** - * jsctx_rb_pull(): - Mark atom in list as running - * @kctx: Pointer to kbase context with ring buffer. - * @katom: Pointer to katom to pull. - * - * Mark an atom previously obtained from jsctx_rb_peek() as running. - * - * @katom must currently be at the head of the ring buffer. - */ -static inline void jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - int prio = katom->sched_priority; - unsigned int js = katom->slot_nr; - struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - dev_dbg(kctx->kbdev->dev, "Erasing atom %pK from runnable tree of kctx %pK\n", - (void *)katom, (void *)kctx); - - /* Atoms must be pulled in the correct order. */ - WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio)); - - rb_erase(&katom->runnable_tree_node, &rb->runnable_tree); -} - -static void jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - struct kbase_device *kbdev = kctx->kbdev; - int prio = katom->sched_priority; - unsigned int js = katom->slot_nr; - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; - struct rb_node **new = &(queue->runnable_tree.rb_node), *parent = NULL; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - dev_dbg(kbdev->dev, "Adding atom %pK to runnable tree of kctx %pK (s:%u)\n", (void *)katom, - (void *)kctx, js); - - while (*new) { - struct kbase_jd_atom *entry = - container_of(*new, struct kbase_jd_atom, runnable_tree_node); - - parent = *new; - if (kbase_jd_atom_is_younger(katom, entry)) - new = &((*new)->rb_left); - else - new = &((*new)->rb_right); - } - - /* Add new node and rebalance tree. */ - rb_link_node(&katom->runnable_tree_node, parent, new); - rb_insert_color(&katom->runnable_tree_node, &queue->runnable_tree); - - KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_READY); -} - -/** - * jsctx_rb_unpull(): - Undo marking of atom in list as running - * @kctx: Pointer to kbase context with ring buffer. - * @katom: Pointer to katom to unpull. - * - * Undo jsctx_rb_pull() and put @katom back in the queue. - * - * jsctx_rb_unpull() must be called on atoms in the same order the atoms were - * pulled. - */ -static inline void jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - KBASE_KTRACE_ADD_JM(kctx->kbdev, JS_UNPULL_JOB, kctx, katom, katom->jc, 0u); - - jsctx_tree_add(kctx, katom); -} - -static bool kbase_js_ctx_pullable(struct kbase_context *kctx, unsigned int js, bool is_scheduled); -static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js); -static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js); - -typedef bool(katom_ordering_func)(const struct kbase_jd_atom *, const struct kbase_jd_atom *); - -bool kbase_js_atom_runs_before(struct kbase_device *kbdev, const struct kbase_jd_atom *katom_a, - const struct kbase_jd_atom *katom_b, - const kbase_atom_ordering_flag_t order_flags) -{ - struct kbase_context *kctx_a = katom_a->kctx; - struct kbase_context *kctx_b = katom_b->kctx; - katom_ordering_func *samectxatomprio_ordering_func = kbase_jd_atom_is_younger; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (order_flags & KBASE_ATOM_ORDERING_FLAG_SEQNR) - samectxatomprio_ordering_func = kbase_jd_atom_is_earlier; - - /* It only makes sense to make this test for atoms on the same slot */ - WARN_ON(katom_a->slot_nr != katom_b->slot_nr); - - if (kbdev->js_ctx_scheduling_mode == KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE) { - /* In local priority mode, querying either way around for "a - * should run before b" and "b should run before a" should - * always be false when they're from different contexts - */ - if (kctx_a != kctx_b) - return false; - } else { - /* In system priority mode, ordering is done first strictly by - * context priority, even when katom_b might be lower priority - * than katom_a. This is due to scheduling of contexts in order - * of highest priority first, regardless of whether the atoms - * for a particular slot from such contexts have the highest - * priority or not. - */ - if (kctx_a != kctx_b) { - if (kctx_a->priority < kctx_b->priority) - return true; - if (kctx_a->priority > kctx_b->priority) - return false; - } - } - - /* For same contexts/contexts with the same context priority (in system - * priority mode), ordering is next done by atom priority - */ - if (katom_a->sched_priority < katom_b->sched_priority) - return true; - if (katom_a->sched_priority > katom_b->sched_priority) - return false; - /* For atoms of same priority on the same kctx, they are - * ordered by seq_nr/age (dependent on caller) - */ - if (kctx_a == kctx_b && samectxatomprio_ordering_func(katom_a, katom_b)) - return true; - - return false; -} - -/* - * Functions private to KBase ('Protected' functions) - */ -int kbasep_js_devdata_init(struct kbase_device *const kbdev) -{ - struct kbasep_js_device_data *jsdd; - int i, j; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - - jsdd = &kbdev->js_data; - - INIT_LIST_HEAD(&jsdd->suspended_soft_jobs_list); - - /* Config attributes */ - jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; - jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; - jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; - jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; - jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; - jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING; - jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; - jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; - - jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING; - jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS; - atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT); - jsdd->js_free_wait_time_ms = kbase_get_timeout_ms(kbdev, JM_DEFAULT_JS_FREE_TIMEOUT); - - dev_dbg(kbdev->dev, "JS Config Attribs: "); - dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u", jsdd->scheduling_period_ns); - dev_dbg(kbdev->dev, "\tsoft_stop_ticks:%u", jsdd->soft_stop_ticks); - dev_dbg(kbdev->dev, "\tsoft_stop_ticks_cl:%u", jsdd->soft_stop_ticks_cl); - dev_dbg(kbdev->dev, "\thard_stop_ticks_ss:%u", jsdd->hard_stop_ticks_ss); - dev_dbg(kbdev->dev, "\thard_stop_ticks_cl:%u", jsdd->hard_stop_ticks_cl); - dev_dbg(kbdev->dev, "\thard_stop_ticks_dumping:%u", jsdd->hard_stop_ticks_dumping); - dev_dbg(kbdev->dev, "\tgpu_reset_ticks_ss:%u", jsdd->gpu_reset_ticks_ss); - dev_dbg(kbdev->dev, "\tgpu_reset_ticks_cl:%u", jsdd->gpu_reset_ticks_cl); - dev_dbg(kbdev->dev, "\tgpu_reset_ticks_dumping:%u", jsdd->gpu_reset_ticks_dumping); - dev_dbg(kbdev->dev, "\tctx_timeslice_ns:%u", jsdd->ctx_timeslice_ns); - dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i", atomic_read(&jsdd->soft_job_timeout_ms)); - dev_dbg(kbdev->dev, "\tjs_free_wait_time_ms:%u", jsdd->js_free_wait_time_ms); - - if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss && - jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss && - jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_dumping && - jsdd->hard_stop_ticks_dumping < jsdd->gpu_reset_ticks_dumping)) { - dev_err(kbdev->dev, - "Job scheduler timeouts invalid; soft/hard/reset tick counts should be in increasing order\n"); - return -EINVAL; - } - -#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS - dev_dbg(kbdev->dev, - "Job Scheduling Soft-stops disabled, ignoring value for soft_stop_ticks==%u at %uns per tick. Other soft-stops may still occur.", - jsdd->soft_stop_ticks, jsdd->scheduling_period_ns); -#endif -#if KBASE_DISABLE_SCHEDULING_HARD_STOPS - dev_dbg(kbdev->dev, - "Job Scheduling Hard-stops disabled, ignoring values for hard_stop_ticks_ss==%d and hard_stop_ticks_dumping==%u at %uns per tick. Other hard-stops may still occur.", - jsdd->hard_stop_ticks_ss, jsdd->hard_stop_ticks_dumping, - jsdd->scheduling_period_ns); -#endif -#if KBASE_DISABLE_SCHEDULING_SOFT_STOPS && KBASE_DISABLE_SCHEDULING_HARD_STOPS - dev_dbg(kbdev->dev, - "Note: The JS tick timer (if coded) will still be run, but do nothing."); -#endif - - for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) - jsdd->js_reqs[i] = core_reqs_from_jsn_features(&kbdev->gpu_props.js_features[i]); - - /* On error, we could continue on: providing none of the below resources - * rely on the ones above - */ - - mutex_init(&jsdd->runpool_mutex); - mutex_init(&jsdd->queue_mutex); - sema_init(&jsdd->schedule_sem, 1); - - for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { - for (j = KBASE_JS_ATOM_SCHED_PRIO_FIRST; j < KBASE_JS_ATOM_SCHED_PRIO_COUNT; ++j) { - INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i][j]); - INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i][j]); - } - } - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - hrtimer_init(&jsdd->gpu_metrics_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - jsdd->gpu_metrics_timer.function = gpu_metrics_timer_callback; - jsdd->gpu_metrics_timer_needed = false; - jsdd->gpu_metrics_timer_running = false; -#endif - - return 0; -} - -void kbasep_js_devdata_halt(struct kbase_device *kbdev) -{ - CSTD_UNUSED(kbdev); -} - -void kbasep_js_devdata_term(struct kbase_device *kbdev) -{ - struct kbasep_js_device_data *js_devdata; - s8 zero_ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT] = { - 0, - }; - CSTD_UNUSED(js_devdata); - - KBASE_DEBUG_ASSERT(kbdev != NULL); - - js_devdata = &kbdev->js_data; - - /* The caller must de-register all contexts before calling this - */ - KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running == 0); - KBASE_DEBUG_ASSERT(memcmp(js_devdata->runpool_irq.ctx_attr_ref_count, - zero_ctx_attr_ref_count, sizeof(zero_ctx_attr_ref_count)) == 0); - CSTD_UNUSED(zero_ctx_attr_ref_count); - -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - js_devdata->gpu_metrics_timer_needed = false; - hrtimer_cancel(&js_devdata->gpu_metrics_timer); -#endif -} - -int kbasep_js_kctx_init(struct kbase_context *const kctx) -{ - struct kbasep_js_kctx_info *js_kctx_info; - int i, j; - int ret; - CSTD_UNUSED(js_kctx_info); - - KBASE_DEBUG_ASSERT(kctx != NULL); - - CSTD_UNUSED(ret); -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - ret = gpu_metrics_ctx_init(kctx); - if (ret) - return ret; -#endif - - for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) - INIT_LIST_HEAD(&kctx->jctx.sched_info.ctx.ctx_list_entry[i]); - - js_kctx_info = &kctx->jctx.sched_info; - - kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); - kbase_ctx_flag_clear(kctx, KCTX_DYING); - - /* Initially, the context is disabled from submission until the create - * flags are set - */ - kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); - - /* On error, we could continue on: providing none of the below resources - * rely on the ones above - */ - mutex_init(&js_kctx_info->ctx.jsctx_mutex); - - init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); - - for (i = KBASE_JS_ATOM_SCHED_PRIO_FIRST; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { - for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { - INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); - kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; - } - } - - return 0; -} - -void kbasep_js_kctx_term(struct kbase_context *kctx) -{ - struct kbase_device *kbdev; - struct kbasep_js_kctx_info *js_kctx_info; - unsigned int js; - bool update_ctx_count = false; - unsigned long flags; - CSTD_UNUSED(js_kctx_info); - - KBASE_DEBUG_ASSERT(kctx != NULL); - - kbdev = kctx->kbdev; - KBASE_DEBUG_ASSERT(kbdev != NULL); - - js_kctx_info = &kctx->jctx.sched_info; - - /* The caller must de-register all jobs before calling this */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs == 0); - - mutex_lock(&kbdev->js_data.queue_mutex); - mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)) { - WARN_ON(atomic_read(&kbdev->js_data.nr_contexts_runnable) <= 0); - atomic_dec(&kbdev->js_data.nr_contexts_runnable); - update_ctx_count = true; - kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); - } - - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - mutex_unlock(&kbdev->js_data.queue_mutex); - - if (update_ctx_count) { - mutex_lock(&kbdev->js_data.runpool_mutex); - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&kbdev->js_data.runpool_mutex); - } - - kbase_ctx_sched_remove_ctx(kctx); -#if IS_ENABLED(CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD) - gpu_metrics_ctx_term(kctx); -#endif -} - -/* - * Priority blocking management functions - */ - -/* Should not normally use directly - use kbase_jsctx_slot_atom_pulled_dec() instead */ -static void kbase_jsctx_slot_prio_blocked_clear(struct kbase_context *kctx, unsigned int js, - int sched_prio) -{ - struct kbase_jsctx_slot_tracking *slot_tracking = &kctx->slot_tracking[js]; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - slot_tracking->blocked &= ~(((kbase_js_prio_bitmap_t)1) << sched_prio); - KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev, JS_SLOT_PRIO_UNBLOCKED, kctx, NULL, 0, js, - (unsigned int)sched_prio); -} - -static int kbase_jsctx_slot_atoms_pulled(struct kbase_context *kctx, unsigned int js) -{ - return atomic_read(&kctx->slot_tracking[js].atoms_pulled); -} - -/* - * A priority level on a slot is blocked when: - * - that priority level is blocked - * - or, any higher priority level is blocked - */ -static bool kbase_jsctx_slot_prio_is_blocked(struct kbase_context *kctx, unsigned int js, - int sched_prio) -{ - struct kbase_jsctx_slot_tracking *slot_tracking = &kctx->slot_tracking[js]; - kbase_js_prio_bitmap_t prio_bit, higher_prios_mask; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - /* done in two separate shifts to prevent future undefined behavior - * should the number of priority levels == (bit width of the type) - */ - prio_bit = (((kbase_js_prio_bitmap_t)1) << sched_prio); - /* all bits of sched_prio or higher, with sched_prio = 0 being the - * highest priority - */ - higher_prios_mask = (prio_bit << 1u) - 1u; - return (slot_tracking->blocked & higher_prios_mask) != 0u; -} - -/** - * kbase_jsctx_slot_atom_pulled_inc - Increase counts of atoms that have being - * pulled for a slot from a ctx, based on - * this atom - * @kctx: kbase context - * @katom: atom pulled - * - * Manages counts of atoms pulled (including per-priority-level counts), for - * later determining when a ctx can become unblocked on a slot. - * - * Once a slot has been blocked at @katom's priority level, it should not be - * pulled from, hence this function should not be called in that case. - * - * The return value is to aid tracking of when @kctx becomes runnable. - * - * Return: new total count of atoms pulled from all slots on @kctx - */ -static int kbase_jsctx_slot_atom_pulled_inc(struct kbase_context *kctx, - const struct kbase_jd_atom *katom) -{ - unsigned int js = katom->slot_nr; - int sched_prio = katom->sched_priority; - struct kbase_jsctx_slot_tracking *slot_tracking = &kctx->slot_tracking[js]; - int nr_atoms_pulled; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - WARN(kbase_jsctx_slot_prio_is_blocked(kctx, js, sched_prio), - "Should not have pulled atoms for slot %u from a context that is blocked at priority %d or higher", - js, sched_prio); - - nr_atoms_pulled = atomic_inc_return(&kctx->atoms_pulled_all_slots); - atomic_inc(&slot_tracking->atoms_pulled); - slot_tracking->atoms_pulled_pri[sched_prio]++; - - return nr_atoms_pulled; -} - -/** - * kbase_jsctx_slot_atom_pulled_dec- Decrease counts of atoms that have being - * pulled for a slot from a ctx, and - * re-evaluate whether a context is blocked - * on this slot - * @kctx: kbase context - * @katom: atom that has just been removed from a job slot - * - * @kctx can become unblocked on a slot for a priority level when it no longer - * has any pulled atoms at that priority level on that slot, and all higher - * (numerically lower) priority levels are also unblocked @kctx on that - * slot. The latter condition is to retain priority ordering within @kctx. - * - * Return: true if the slot was previously blocked but has now become unblocked - * at @katom's priority level, false otherwise. - */ -static bool kbase_jsctx_slot_atom_pulled_dec(struct kbase_context *kctx, - const struct kbase_jd_atom *katom) -{ - unsigned int js = katom->slot_nr; - int sched_prio = katom->sched_priority; - int atoms_pulled_pri; - struct kbase_jsctx_slot_tracking *slot_tracking = &kctx->slot_tracking[js]; - bool slot_prio_became_unblocked = false; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - atomic_dec(&kctx->atoms_pulled_all_slots); - atomic_dec(&slot_tracking->atoms_pulled); - - atoms_pulled_pri = --(slot_tracking->atoms_pulled_pri[sched_prio]); - - /* We can safely clear this priority level's blocked status even if - * higher priority levels are still blocked: a subsequent query to - * kbase_jsctx_slot_prio_is_blocked() will still return true - */ - if (!atoms_pulled_pri && kbase_jsctx_slot_prio_is_blocked(kctx, js, sched_prio)) { - kbase_jsctx_slot_prio_blocked_clear(kctx, js, sched_prio); - - if (!kbase_jsctx_slot_prio_is_blocked(kctx, js, sched_prio)) - slot_prio_became_unblocked = true; - } - - if (slot_prio_became_unblocked) - KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev, JS_SLOT_PRIO_AND_HIGHER_UNBLOCKED, kctx, - katom, katom->jc, js, (unsigned int)sched_prio); - - return slot_prio_became_unblocked; -} - -/** - * kbase_js_ctx_list_add_pullable_nolock - Variant of - * kbase_jd_ctx_list_add_pullable() - * where the caller must hold - * hwaccess_lock - * @kbdev: Device pointer - * @kctx: Context to add to queue - * @js: Job slot to use - * - * Caller must hold hwaccess_lock - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js) -{ - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "Add pullable tail kctx %pK (s:%u)\n", (void *)kctx, js); - - if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - - list_add_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], - &kbdev->js_data.ctx_list_pullable[js][kctx->priority]); - - if (!kctx->slots_pullable) { - kbdev->js_data.nr_contexts_pullable++; - ret = true; - if (!kbase_jsctx_atoms_pulled(kctx)) { - WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); - atomic_inc(&kbdev->js_data.nr_contexts_runnable); - } - } - kctx->slots_pullable |= (1UL << js); - - return ret; -} - -/** - * kbase_js_ctx_list_add_pullable_head_nolock - Variant of - * kbase_js_ctx_list_add_pullable_head() - * where the caller must hold - * hwaccess_lock - * @kbdev: Device pointer - * @kctx: Context to add to queue - * @js: Job slot to use - * - * Caller must hold hwaccess_lock - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_add_pullable_head_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js) -{ - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "Add pullable head kctx %pK (s:%u)\n", (void *)kctx, js); - - if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - - list_add(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], - &kbdev->js_data.ctx_list_pullable[js][kctx->priority]); - - if (!kctx->slots_pullable) { - kbdev->js_data.nr_contexts_pullable++; - ret = true; - if (!kbase_jsctx_atoms_pulled(kctx)) { - WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); - atomic_inc(&kbdev->js_data.nr_contexts_runnable); - } - } - kctx->slots_pullable |= (1UL << js); - - return ret; -} - -/** - * kbase_js_ctx_list_add_pullable_head - Add context to the head of the - * per-slot pullable context queue - * @kbdev: Device pointer - * @kctx: Context to add to queue - * @js: Job slot to use - * - * If the context is on either the pullable or unpullable queues, then it is - * removed before being added to the head. - * - * This function should be used when a context has been scheduled, but no jobs - * can currently be pulled from it. - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js) -{ - bool ret; - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - ret = kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, js); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return ret; -} - -/** - * kbase_js_ctx_list_add_unpullable_nolock - Add context to the tail of the - * per-slot unpullable context queue - * @kbdev: Device pointer - * @kctx: Context to add to queue - * @js: Job slot to use - * - * The context must already be on the per-slot pullable queue. It will be - * removed from the pullable queue before being added to the unpullable queue. - * - * This function should be used when a context has been pulled from, and there - * are no jobs remaining on the specified slot. - * - * Caller must hold hwaccess_lock - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, unsigned int js) -{ - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "Add unpullable tail kctx %pK (s:%u)\n", (void *)kctx, js); - - list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], - &kbdev->js_data.ctx_list_unpullable[js][kctx->priority]); - - if (kctx->slots_pullable == (1UL << js)) { - kbdev->js_data.nr_contexts_pullable--; - ret = true; - if (!kbase_jsctx_atoms_pulled(kctx)) { - WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); - atomic_dec(&kbdev->js_data.nr_contexts_runnable); - } - } - kctx->slots_pullable &= ~(1UL << js); - - return ret; -} - -/** - * kbase_js_ctx_list_remove_nolock - Remove context from the per-slot pullable - * or unpullable context queues - * @kbdev: Device pointer - * @kctx: Context to remove from queue - * @js: Job slot to use - * - * The context must already be on one of the queues. - * - * This function should be used when a context has no jobs on the GPU, and no - * jobs remaining for the specified slot. - * - * Caller must hold hwaccess_lock - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js) -{ - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])); - - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - - if (kctx->slots_pullable == (1UL << js)) { - kbdev->js_data.nr_contexts_pullable--; - ret = true; - if (!kbase_jsctx_atoms_pulled(kctx)) { - WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); - atomic_dec(&kbdev->js_data.nr_contexts_runnable); - } - } - kctx->slots_pullable &= ~(1UL << js); - - return ret; -} - -/** - * kbase_js_ctx_list_pop_head_nolock - Variant of kbase_js_ctx_list_pop_head() - * where the caller must hold - * hwaccess_lock - * @kbdev: Device pointer - * @js: Job slot to use - * - * Caller must hold hwaccess_lock - * - * Return: Context to use for specified slot. - * NULL if no contexts present for specified slot - */ -static struct kbase_context *kbase_js_ctx_list_pop_head_nolock(struct kbase_device *kbdev, - unsigned int js) -{ - struct kbase_context *kctx; - int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = KBASE_JS_ATOM_SCHED_PRIO_FIRST; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { - if (list_empty(&kbdev->js_data.ctx_list_pullable[js][i])) - continue; - - kctx = list_entry(kbdev->js_data.ctx_list_pullable[js][i].next, - struct kbase_context, jctx.sched_info.ctx.ctx_list_entry[js]); - - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - dev_dbg(kbdev->dev, "Popped %pK from the pullable queue (s:%u)\n", (void *)kctx, - js); - return kctx; - } - return NULL; -} - -/** - * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable - * queue. - * @kbdev: Device pointer - * @js: Job slot to use - * - * Return: Context to use for specified slot. - * NULL if no contexts present for specified slot - */ -static struct kbase_context *kbase_js_ctx_list_pop_head(struct kbase_device *kbdev, unsigned int js) -{ - struct kbase_context *kctx; - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kctx = kbase_js_ctx_list_pop_head_nolock(kbdev, js); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return kctx; -} - -/** - * kbase_js_ctx_pullable - Return if a context can be pulled from on the - * specified slot - * @kctx: Context pointer - * @js: Job slot to use - * @is_scheduled: true if the context is currently scheduled - * - * Caller must hold hwaccess_lock - * - * Return: true if context can be pulled from on specified slot - * false otherwise - */ -static bool kbase_js_ctx_pullable(struct kbase_context *kctx, unsigned int js, bool is_scheduled) -{ - struct kbasep_js_device_data *js_devdata; - struct kbase_jd_atom *katom; - struct kbase_device *kbdev = kctx->kbdev; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - js_devdata = &kbdev->js_data; - - if (is_scheduled) { - if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) { - dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %pK\n", (void *)kctx); - return false; - } - } - katom = jsctx_rb_peek(kctx, js); - if (!katom) { - dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %pK (s:%u)\n", (void *)kctx, js); - return false; /* No pullable atoms */ - } - if (kbase_jsctx_slot_prio_is_blocked(kctx, js, katom->sched_priority)) { - KBASE_KTRACE_ADD_JM_SLOT_INFO(kctx->kbdev, JS_SLOT_PRIO_IS_BLOCKED, kctx, katom, - katom->jc, js, (unsigned int)katom->sched_priority); - dev_dbg(kbdev->dev, - "JS: kctx %pK is blocked from submitting atoms at priority %d and lower (s:%u)\n", - (void *)kctx, katom->sched_priority, js); - return false; - } - if (atomic_read(&katom->blocked)) { - dev_dbg(kbdev->dev, "JS: Atom %pK is blocked in js_ctx_pullable\n", (void *)katom); - return false; /* next atom blocked */ - } - if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { - if (katom->x_pre_dep->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || - katom->x_pre_dep->will_fail_event_code) { - dev_dbg(kbdev->dev, - "JS: X pre-dep %pK is not present in slot FIFO or will fail\n", - (void *)katom->x_pre_dep); - return false; - } - if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && - kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) { - dev_dbg(kbdev->dev, - "JS: Atom %pK has cross-slot fail dependency and atoms on slot (s:%u)\n", - (void *)katom, js); - return false; - } - } - - dev_dbg(kbdev->dev, "JS: Atom %pK is pullable in kctx %pK (s:%u)\n", (void *)katom, - (void *)kctx, js); - - return true; -} - -static bool kbase_js_dep_validate(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - struct kbase_device *kbdev = kctx->kbdev; - bool ret = true; - bool has_dep = false, has_x_dep = false; - unsigned int js = kbase_js_get_slot(kbdev, katom); - int prio = katom->sched_priority; - int i; - - for (i = 0; i < 2; i++) { - struct kbase_jd_atom *dep_atom = katom->dep[i].atom; - - if (dep_atom) { - unsigned int dep_js = kbase_js_get_slot(kbdev, dep_atom); - int dep_prio = dep_atom->sched_priority; - - dev_dbg(kbdev->dev, "Checking dep %d of atom %pK (s:%d) on %pK (s:%d)\n", i, - (void *)katom, js, (void *)dep_atom, dep_js); - - /* Dependent atom must already have been submitted */ - if (!(dep_atom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { - dev_dbg(kbdev->dev, "Blocker not submitted yet\n"); - ret = false; - break; - } - - /* Dependencies with different priorities can't - * be represented in the ringbuffer - */ - if (prio != dep_prio) { - dev_dbg(kbdev->dev, "Different atom priorities\n"); - ret = false; - break; - } - - if (js == dep_js) { - /* Only one same-slot dependency can be - * represented in the ringbuffer - */ - if (has_dep) { - dev_dbg(kbdev->dev, "Too many same-slot deps\n"); - ret = false; - break; - } - /* Each dependee atom can only have one - * same-slot dependency - */ - if (dep_atom->post_dep) { - dev_dbg(kbdev->dev, "Too many same-slot successors\n"); - ret = false; - break; - } - has_dep = true; - } else { - /* Only one cross-slot dependency can be - * represented in the ringbuffer - */ - if (has_x_dep) { - dev_dbg(kbdev->dev, "Too many cross-slot deps\n"); - ret = false; - break; - } - /* Each dependee atom can only have one - * cross-slot dependency - */ - if (dep_atom->x_post_dep) { - dev_dbg(kbdev->dev, "Too many cross-slot successors\n"); - ret = false; - break; - } - /* The dependee atom can not already be in the - * HW access ringbuffer - */ - if (dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - dev_dbg(kbdev->dev, - "Blocker already in ringbuffer (state:%d)\n", - dep_atom->gpu_rb_state); - ret = false; - break; - } - /* The dependee atom can not already have - * completed - */ - if (dep_atom->status != KBASE_JD_ATOM_STATE_IN_JS) { - dev_dbg(kbdev->dev, - "Blocker already completed (status:%d)\n", - dep_atom->status); - ret = false; - break; - } - - has_x_dep = true; - } - - /* Dependency can be represented in ringbuffers */ - } - } - - /* If dependencies can be represented by ringbuffer then clear them from - * atom structure - */ - if (ret) { - for (i = 0; i < 2; i++) { - struct kbase_jd_atom *dep_atom = katom->dep[i].atom; - - if (dep_atom) { - unsigned int dep_js = kbase_js_get_slot(kbdev, dep_atom); - - dev_dbg(kbdev->dev, - "Clearing dep %d of atom %pK (s:%d) on %pK (s:%d)\n", i, - (void *)katom, js, (void *)dep_atom, dep_js); - - if ((js != dep_js) && - (dep_atom->status != KBASE_JD_ATOM_STATE_COMPLETED) && - (dep_atom->status != KBASE_JD_ATOM_STATE_HW_COMPLETED) && - (dep_atom->status != KBASE_JD_ATOM_STATE_UNUSED)) { - katom->atom_flags |= KBASE_KATOM_FLAG_X_DEP_BLOCKED; - - katom->x_pre_dep = dep_atom; - dep_atom->x_post_dep = katom; - if (kbase_jd_katom_dep_type(&katom->dep[i]) == - BASE_JD_DEP_TYPE_DATA) - katom->atom_flags |= KBASE_KATOM_FLAG_FAIL_BLOCKER; - } - if ((kbase_jd_katom_dep_type(&katom->dep[i]) == - BASE_JD_DEP_TYPE_DATA) && - (js == dep_js)) { - katom->pre_dep = dep_atom; - dep_atom->post_dep = katom; - } - - list_del(&katom->dep_item[i]); - kbase_jd_katom_dep_clear(&katom->dep[i]); - } - } - } else { - dev_dbg(kbdev->dev, "Deps of atom %pK (s:%d) could not be represented\n", - (void *)katom, js); - } - - return ret; -} - -void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority) -{ - struct kbase_device *kbdev = kctx->kbdev; - unsigned int js; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* Move kctx to the pullable/upullable list as per the new priority */ - if (new_priority != kctx->priority) { - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (kctx->slots_pullable & (1UL << js)) - list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], - &kbdev->js_data.ctx_list_pullable[js][new_priority]); - else - list_move_tail( - &kctx->jctx.sched_info.ctx.ctx_list_entry[js], - &kbdev->js_data.ctx_list_unpullable[js][new_priority]); - } - - kctx->priority = new_priority; - } -} - -void kbase_js_update_ctx_priority(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - int new_priority = KBASE_JS_ATOM_SCHED_PRIO_LOW; - int prio; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (kbdev->js_ctx_scheduling_mode == KBASE_JS_SYSTEM_PRIORITY_MODE) { - /* Determine the new priority for context, as per the priority - * of currently in-use atoms. - */ - for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; - prio++) { - if (kctx->atoms_count[prio]) { - new_priority = prio; - break; - } - } - } - - kbase_js_set_ctx_priority(kctx, new_priority); -} -KBASE_EXPORT_TEST_API(kbase_js_update_ctx_priority); - -bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom) -{ - unsigned long flags; - struct kbasep_js_kctx_info *js_kctx_info; - struct kbase_device *kbdev; - struct kbasep_js_device_data *js_devdata; - - bool enqueue_required = false; - bool timer_sync = false; - - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(atom != NULL); - lockdep_assert_held(&kctx->jctx.lock); - - kbdev = kctx->kbdev; - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - - /* - * Begin Runpool transaction - */ - mutex_lock(&js_devdata->runpool_mutex); - - /* Refcount ctx.nr_jobs */ - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX); - ++(js_kctx_info->ctx.nr_jobs); - dev_dbg(kbdev->dev, "Add atom %pK to kctx %pK; now %d in ctx\n", (void *)atom, (void *)kctx, - js_kctx_info->ctx.nr_jobs); - - /* Lock for state available during IRQ */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (++kctx->atoms_count[atom->sched_priority] == 1) - kbase_js_update_ctx_priority(kctx); - - if (!kbase_js_dep_validate(kctx, atom)) { - /* Dependencies could not be represented */ - --(js_kctx_info->ctx.nr_jobs); - dev_dbg(kbdev->dev, "Remove atom %pK from kctx %pK; now %d in ctx\n", (void *)atom, - (void *)kctx, js_kctx_info->ctx.nr_jobs); - - /* Setting atom status back to queued as it still has unresolved - * dependencies - */ - atom->status = KBASE_JD_ATOM_STATE_QUEUED; - dev_dbg(kbdev->dev, "Atom %pK status to queued\n", (void *)atom); - - /* Undo the count, as the atom will get added again later but - * leave the context priority adjusted or boosted, in case if - * this was the first higher priority atom received for this - * context. - * This will prevent the scenario of priority inversion, where - * another context having medium priority atoms keeps getting - * scheduled over this context, which is having both lower and - * higher priority atoms, but higher priority atoms are blocked - * due to dependency on lower priority atoms. With priority - * boost the high priority atom will get to run at earliest. - */ - kctx->atoms_count[atom->sched_priority]--; - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_devdata->runpool_mutex); - - goto out_unlock; - } - - enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); - - KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, - kbase_ktrace_get_ctx_refcnt(kctx)); - - /* Context Attribute Refcounting */ - kbasep_js_ctx_attr_ctx_retain_atom(kbdev, kctx, atom); - - if (enqueue_required) { - if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false)) - timer_sync = - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, atom->slot_nr); - else - timer_sync = - kbase_js_ctx_list_add_unpullable_nolock(kbdev, kctx, atom->slot_nr); - } - /* If this context is active and the atom is the first on its slot, - * kick the job manager to attempt to fast-start the atom - */ - if (enqueue_required && kctx == kbdev->hwaccess.active_kctx[atom->slot_nr]) - kbase_jm_try_kick(kbdev, 1UL << atom->slot_nr); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (timer_sync) - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&js_devdata->runpool_mutex); - /* End runpool transaction */ - - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { - if (kbase_ctx_flag(kctx, KCTX_DYING)) { - /* A job got added while/after kbase_job_zap_context() - * was called on a non-scheduled context. Kill that job - * by killing the context. - */ - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); - } else if (js_kctx_info->ctx.nr_jobs == 1) { - /* Handle Refcount going from 0 to 1: schedule the - * context on the Queue - */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - dev_dbg(kbdev->dev, "JS: Enqueue Context %pK", kctx); - - /* Queue was updated - caller must try to schedule the - * head context - */ - WARN_ON(!enqueue_required); - } - } -out_unlock: - dev_dbg(kbdev->dev, "Enqueue of kctx %pK is %srequired\n", kctx, - enqueue_required ? "" : "not "); - - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - mutex_unlock(&js_devdata->queue_mutex); - - return enqueue_required; -} - -void kbasep_js_remove_job(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *atom) -{ - struct kbasep_js_kctx_info *js_kctx_info; - unsigned long flags; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(atom != NULL); - - js_kctx_info = &kctx->jctx.sched_info; - - KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_REMOVE_JOB, kctx, atom, atom->jc, - kbase_ktrace_get_ctx_refcnt(kctx)); - - /* De-refcount ctx.nr_jobs */ - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); - --(js_kctx_info->ctx.nr_jobs); - dev_dbg(kbdev->dev, "Remove atom %pK from kctx %pK; now %d in ctx\n", (void *)atom, - (void *)kctx, js_kctx_info->ctx.nr_jobs); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (--kctx->atoms_count[atom->sched_priority] == 0) - kbase_js_update_ctx_priority(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *katom) -{ - unsigned long flags; - struct kbasep_js_atom_retained_state katom_retained_state; - bool attr_state_changed; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(katom != NULL); - - kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); - kbasep_js_remove_job(kbdev, kctx, katom); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* The atom has 'finished' (will not be re-run), so no need to call - * kbasep_js_has_atom_finished(). - * - * This is because it returns false for soft-stopped atoms, but we - * want to override that, because we're cancelling an atom regardless of - * whether it was soft-stopped or not - */ - attr_state_changed = - kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, &katom_retained_state); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return attr_state_changed; -} - -/** - * kbasep_js_run_jobs_after_ctx_and_atom_release - Try running more jobs after - * releasing a context and/or atom - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * @katom_retained_state: Retained state from the atom - * @runpool_ctx_attr_change: True if the runpool context attributes have changed - * - * This collates a set of actions that must happen whilst hwaccess_lock is held. - * - * This includes running more jobs when: - * - The previously released kctx caused a ctx attribute change, - * - The released atom caused a ctx attribute change, - * - Slots were previously blocked due to affinity restrictions, - * - Submission during IRQ handling failed. - * - * Return: %KBASEP_JS_RELEASE_RESULT_SCHED_ALL if context attributes were - * changed. The caller should try scheduling all contexts - */ -static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( - struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state, bool runpool_ctx_attr_change) -{ - struct kbasep_js_device_data *js_devdata; - kbasep_js_release_result result = 0; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(katom_retained_state != NULL); - js_devdata = &kbdev->js_data; - - lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); - lockdep_assert_held(&js_devdata->runpool_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (js_devdata->nr_user_contexts_running != 0 && runpool_ctx_attr_change) { - /* A change in runpool ctx attributes might mean we can - * run more jobs than before - */ - result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; - - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, kctx, NULL, 0u, 0); - } - return result; -} - -/** - * kbasep_js_runpool_release_ctx_internal - Internal function to release the reference - * on a ctx and an atom's "retained state", only - * taking the runpool and as transaction mutexes - * @kbdev: The kbase_device to operate on - * @kctx: The kbase_context to operate on - * @katom_retained_state: Retained state from the atom - * - * This also starts more jobs running in the case of an ctx-attribute state change - * - * This does none of the followup actions for scheduling: - * - It does not schedule in a new context - * - It does not requeue or handle dying contexts - * - * For those tasks, just call kbasep_js_runpool_release_ctx() instead - * - * Has following requirements - * - Context is scheduled in, and kctx->as_nr matches kctx_as_nr - * - Context has a non-zero refcount - * - Caller holds js_kctx_info->ctx.jsctx_mutex - * - Caller holds js_devdata->runpool_mutex - * - * Return: A bitpattern, containing KBASEP_JS_RELEASE_RESULT_* flags, indicating - * the result of releasing a context that whether the caller should try - * scheduling a new context or should try scheduling all contexts. - */ -static kbasep_js_release_result -kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state) -{ - unsigned long flags; - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - - kbasep_js_release_result release_result = 0u; - bool runpool_ctx_attr_change = false; - int kctx_as_nr; - int new_ref_count; - CSTD_UNUSED(kctx_as_nr); - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - - /* Ensure context really is scheduled in */ - KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - kctx_as_nr = kctx->as_nr; - KBASE_DEBUG_ASSERT(kctx_as_nr != KBASEP_AS_NR_INVALID); - KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); - - /* - * Transaction begins on AS and runpool_irq - * - * Assert about out calling contract - */ - mutex_lock(&kbdev->pm.lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - KBASE_DEBUG_ASSERT(kctx_as_nr == kctx->as_nr); - KBASE_DEBUG_ASSERT(atomic_read(&kctx->refcount) > 0); - - /* Update refcount */ - kbase_ctx_sched_release_ctx(kctx); - new_ref_count = atomic_read(&kctx->refcount); - - /* Release the atom if it finished (i.e. wasn't soft-stopped) */ - if (kbasep_js_has_atom_finished(katom_retained_state)) - runpool_ctx_attr_change |= - kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, katom_retained_state); - - if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && - !kbase_io_is_aw_removed(kbdev) && !kbase_pm_is_suspending(kbdev)) { - /* Context is kept scheduled into an address space even when - * there are no jobs, in this case we have to handle the - * situation where all jobs have been evicted from the GPU and - * submission is disabled. - * - * At this point we re-enable submission to allow further jobs - * to be executed - */ - kbasep_js_set_submit_allowed(js_devdata, kctx); - } - - /* Make a set of checks to see if the context should be scheduled out. - * Note that there'll always be at least 1 reference to the context - * which was previously acquired by kbasep_js_schedule_ctx(). - */ - if (new_ref_count == 1 && - (!kbasep_js_is_submit_allowed(js_devdata, kctx) || kbase_io_is_aw_removed(kbdev) || - kbase_pm_is_suspending(kbdev))) { - int num_slots = kbdev->gpu_props.num_job_slots; - unsigned int slot; - - /* Last reference, and we've been told to remove this context - * from the Run Pool - */ - dev_dbg(kbdev->dev, - "JS: RunPool Remove Context %pK because refcount=%d, jobs=%d, allowed=%d", - kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, - kbasep_js_is_submit_allowed(js_devdata, kctx)); - - KBASE_TLSTREAM_TL_NRET_AS_CTX(kbdev, &kbdev->as[kctx->as_nr], kctx); - - kbase_backend_release_ctx_irq(kbdev, kctx); - - for (slot = 0; slot < num_slots; slot++) { - if (kbdev->hwaccess.active_kctx[slot] == kctx) { - dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%d)\n", - (void *)kctx, slot); - kbdev->hwaccess.active_kctx[slot] = NULL; - } - } - - /* Ctx Attribute handling - * - * Releasing atoms attributes must either happen before this, or - * after the KCTX_SHEDULED flag is changed, otherwise we - * double-decount the attributes - */ - runpool_ctx_attr_change |= kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); - - /* Releasing the context and katom retained state can allow - * more jobs to run - */ - release_result |= kbasep_js_run_jobs_after_ctx_and_atom_release( - kbdev, kctx, katom_retained_state, runpool_ctx_attr_change); - - /* - * Transaction ends on AS and runpool_irq: - * - * By this point, the AS-related data is now clear and ready - * for re-use. - * - * Since releases only occur once for each previous successful - * retain, and no more retains are allowed on this context, no - * other thread will be operating in this - * code whilst we are - */ - - /* Recalculate pullable status for all slots */ - for (slot = 0; slot < num_slots; slot++) { - if (kbase_js_ctx_pullable(kctx, slot, false)) - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, slot); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - kbase_backend_release_ctx_noirq(kbdev, kctx); - - mutex_unlock(&kbdev->pm.lock); - - /* Note: Don't reuse kctx_as_nr now */ - - /* Synchronize with any timers */ - kbase_backend_ctx_count_changed(kbdev); - - /* update book-keeping info */ - kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); - /* Signal any waiter that the context is not scheduled, so is - * safe for termination - once the jsctx_mutex is also dropped, - * and jobs have finished. - */ - wake_up(&js_kctx_info->ctx.is_scheduled_wait); - - /* Queue an action to occur after we've dropped the lock */ - release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED | - KBASEP_JS_RELEASE_RESULT_SCHED_ALL; - } else { - kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, katom_retained_state, - runpool_ctx_attr_change); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->pm.lock); - } - - return release_result; -} - -void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - struct kbasep_js_atom_retained_state katom_retained_state; - - /* Setup a dummy katom_retained_state */ - kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); - - kbasep_js_runpool_release_ctx_internal(kbdev, kctx, &katom_retained_state); -} - -void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, - bool has_pm_ref) -{ - CSTD_UNUSED(has_pm_ref); - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - - /* This is called if and only if you've you've detached the context from - * the Runpool Queue, and not added it back to the Runpool - */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - if (kbase_ctx_flag(kctx, KCTX_DYING)) { - /* Dying: don't requeue, but kill all jobs on the context. This - * happens asynchronously - */ - dev_dbg(kbdev->dev, "JS: ** Killing Context %pK on RunPool Remove **", kctx); - kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); - } -} - -void kbasep_js_runpool_release_ctx_and_katom_retained_state( - struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - kbasep_js_release_result release_result; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - - release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, katom_retained_state); - - /* Drop the runpool mutex to allow requeing kctx */ - mutex_unlock(&js_devdata->runpool_mutex); - - if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); - - /* Drop the jsctx_mutex to allow scheduling in a new context */ - - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - - if (release_result & KBASEP_JS_RELEASE_RESULT_SCHED_ALL) - kbase_js_sched_all(kbdev); -} - -void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - struct kbasep_js_atom_retained_state katom_retained_state; - - kbasep_js_atom_retained_state_init_invalid(&katom_retained_state); - - kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); -} - -/* Variant of kbasep_js_runpool_release_ctx() that doesn't call into - * kbase_js_sched_all() - */ -static void kbasep_js_runpool_release_ctx_no_schedule(struct kbase_device *kbdev, - struct kbase_context *kctx) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - kbasep_js_release_result release_result; - struct kbasep_js_atom_retained_state katom_retained_state_struct; - struct kbasep_js_atom_retained_state *katom_retained_state = &katom_retained_state_struct; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - kbasep_js_atom_retained_state_init_invalid(katom_retained_state); - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - - release_result = kbasep_js_runpool_release_ctx_internal(kbdev, kctx, katom_retained_state); - - /* Drop the runpool mutex to allow requeing kctx */ - mutex_unlock(&js_devdata->runpool_mutex); - if ((release_result & KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED) != 0u) - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, true); - - /* Drop the jsctx_mutex to allow scheduling in a new context */ - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - /* NOTE: could return release_result if the caller would like to know - * whether it should schedule a new context, but currently no callers do - */ -} - -void kbase_js_set_timeouts(struct kbase_device *kbdev) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - kbase_backend_timeouts_changed(kbdev); -} - -static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - unsigned long flags; - bool kctx_suspended = false; - int as_nr; - - dev_dbg(kbdev->dev, "Scheduling kctx %pK (s:%u)\n", kctx, js); - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - /* Pick available address space for this context */ - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - as_nr = kbase_ctx_sched_retain_ctx(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - if (as_nr == KBASEP_AS_NR_INVALID) { - as_nr = kbase_backend_find_and_release_free_address_space(kbdev, kctx); - if (as_nr != KBASEP_AS_NR_INVALID) { - /* Attempt to retain the context again, this should - * succeed - */ - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - as_nr = kbase_ctx_sched_retain_ctx(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - - WARN_ON(as_nr == KBASEP_AS_NR_INVALID); - } - } - if ((as_nr < 0) || (as_nr >= BASE_MAX_NR_AS)) - return false; /* No address space currently available */ - - /* - * Atomic transaction on the Context and Run Pool begins - */ - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* Check to see if context is dying due to kbase_job_zap_context() */ - if (kbase_ctx_flag(kctx, KCTX_DYING)) { - /* Roll back the transaction so far and return */ - kbase_ctx_sched_release_ctx(kctx); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return false; - } - - KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_TRY_SCHEDULE_HEAD_CTX, kctx, NULL, 0u, - kbase_ktrace_get_ctx_refcnt(kctx)); - - kbase_ctx_flag_set(kctx, KCTX_SCHEDULED); - - /* Assign context to previously chosen address space */ - if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { - /* Roll back the transaction so far and return */ - kbase_ctx_sched_release_ctx(kctx); - kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - return false; - } - - kbdev->hwaccess.active_kctx[js] = kctx; - - KBASE_TLSTREAM_TL_RET_AS_CTX(kbdev, &kbdev->as[kctx->as_nr], kctx); - - /* Cause any future waiter-on-termination to wait until the context is - * descheduled - */ - wake_up(&js_kctx_info->ctx.is_scheduled_wait); - - /* Re-check for suspending: a suspend could've occurred, and all the - * contexts could've been removed from the runpool before we took this - * lock. In this case, we don't want to allow this context to run jobs, - * we just want it out immediately. - * - * The DMB required to read the suspend flag was issued recently as part - * of the hwaccess_lock locking. If a suspend occurs *after* that lock - * was taken (i.e. this condition doesn't execute), then the - * kbasep_js_suspend() code will cleanup this context instead (by virtue - * of it being called strictly after the suspend flag is set, and will - * wait for this lock to drop) - */ - if (kbase_pm_is_suspending(kbdev) || kbase_io_is_aw_removed(kbdev)) { - /* Cause it to leave at some later point */ - bool retained; - CSTD_UNUSED(retained); - - retained = kbase_ctx_sched_inc_refcount_nolock(kctx); - KBASE_DEBUG_ASSERT(retained); - - kbasep_js_clear_submit_allowed(js_devdata, kctx); - kctx_suspended = true; - } - - kbase_ctx_flag_clear(kctx, KCTX_PULLED_SINCE_ACTIVE_JS0 << js); - - /* Transaction complete */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); - - /* Synchronize with any timers */ - kbase_backend_ctx_count_changed(kbdev); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - /* Note: after this point, the context could potentially get scheduled - * out immediately - */ - - if (kctx_suspended) { - /* Finishing forcing out the context due to a suspend. Use a - * variant of kbasep_js_runpool_release_ctx() that doesn't - * schedule a new context, to prevent a risk of recursion back - * into this function - */ - kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); - return false; - } - return true; -} - -static bool kbase_js_use_ctx(struct kbase_device *kbdev, struct kbase_context *kctx, - unsigned int js) -{ - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && kbase_backend_use_ctx_sched(kbdev, kctx, js)) { - dev_dbg(kbdev->dev, "kctx %pK already has ASID - mark as active (s:%u)\n", - (void *)kctx, js); - - if (kbdev->hwaccess.active_kctx[js] != kctx) { - kbdev->hwaccess.active_kctx[js] = kctx; - kbase_ctx_flag_clear(kctx, KCTX_PULLED_SINCE_ACTIVE_JS0 << js); - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return true; /* Context already scheduled */ - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return kbasep_js_schedule_ctx(kbdev, kctx, js); -} - -void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - struct kbasep_js_kctx_info *js_kctx_info; - struct kbasep_js_device_data *js_devdata; - bool is_scheduled; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - /* This should only happen in response to a system call - * from a user-space thread. - * In a non-arbitrated environment this can never happen - * whilst suspending. - * - * In an arbitrated environment, user-space threads can run - * while we are suspended (for example GPU not available - * to this VM), however in that case we will block on - * the wait event for KCTX_SCHEDULED, since no context - * can be scheduled until we have the GPU again. - */ - if (!kbase_has_arbiter(kbdev)) { - if (WARN_ON(kbase_pm_is_suspending(kbdev))) - return; - } - - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - - /* Mark the context as privileged */ - kbase_ctx_flag_set(kctx, KCTX_PRIVILEGED); - - is_scheduled = kbase_ctx_flag(kctx, KCTX_SCHEDULED); - if (!is_scheduled) { - /* Add the context to the pullable list */ - if (kbase_js_ctx_list_add_pullable_head(kbdev, kctx, 0)) - kbase_js_sync_timers(kbdev); - - /* Fast-starting requires the jsctx_mutex to be dropped, - * because it works on multiple ctxs - */ - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - - /* Try to schedule the context in */ - kbase_js_sched_all(kbdev); - - /* Wait for the context to be scheduled in */ - wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait, - kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - } else { - /* Already scheduled in - We need to retain it to keep the - * corresponding address space - */ - WARN_ON(!kbase_ctx_sched_inc_refcount(kctx)); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - } -} -KBASE_EXPORT_TEST_API(kbasep_js_schedule_privileged_ctx); - -void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - struct kbasep_js_kctx_info *js_kctx_info; - - KBASE_DEBUG_ASSERT(kctx != NULL); - js_kctx_info = &kctx->jctx.sched_info; - - /* We don't need to use the address space anymore */ - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - kbase_ctx_flag_clear(kctx, KCTX_PRIVILEGED); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - /* Release the context - it will be scheduled out */ - kbasep_js_runpool_release_ctx(kbdev, kctx); - - kbase_js_sched_all(kbdev); -} -KBASE_EXPORT_TEST_API(kbasep_js_release_privileged_ctx); - -void kbasep_js_suspend(struct kbase_device *kbdev) -{ - unsigned long flags; - struct kbasep_js_device_data *js_devdata; - int i; - u16 retained = 0u; - - KBASE_DEBUG_ASSERT(kbdev); - KBASE_DEBUG_ASSERT(kbase_pm_is_suspending(kbdev)); - js_devdata = &kbdev->js_data; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* Prevent all contexts from submitting */ - js_devdata->runpool_irq.submit_allowed = 0; - - /* Retain each of the contexts, so we can cause it to leave even if it - * had no refcount to begin with - */ - for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { - struct kbase_context *kctx = kbdev->as_to_kctx[i]; - - retained = retained << 1; - - if (kctx && !(kbdev->as_free & (1u << i))) { - kbase_ctx_sched_retain_ctx_refcount(kctx); - retained |= 1u; - /* This loop will not have an effect on the privileged - * contexts as they would have an extra ref count - * compared to the normal contexts, so they will hold - * on to their address spaces. MMU will re-enabled for - * them on resume. - */ - } - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* De-ref the previous retain to ensure each context gets pulled out - * sometime later. - */ - for (i = 0; i < BASE_MAX_NR_AS; ++i, retained = retained >> 1) { - struct kbase_context *kctx = kbdev->as_to_kctx[i]; - - if (retained & 1u) - kbasep_js_runpool_release_ctx(kbdev, kctx); - } - - /* Caller must wait for all Power Manager active references to be - * dropped - */ -} - -void kbasep_js_resume(struct kbase_device *kbdev) -{ - struct kbasep_js_device_data *js_devdata; - unsigned int js; - int prio; - - KBASE_DEBUG_ASSERT(kbdev); - js_devdata = &kbdev->js_data; - KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); - - mutex_lock(&js_devdata->queue_mutex); - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; - prio++) { - struct kbase_context *kctx, *n; - unsigned long flags; - - if (kbase_has_arbiter(kbdev)) { - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - list_for_each_entry_safe( - kctx, n, &kbdev->js_data.ctx_list_unpullable[js][prio], - jctx.sched_info.ctx.ctx_list_entry[js]) { - struct kbasep_js_kctx_info *js_kctx_info; - bool timer_sync = false; - - /* Drop lock so we can take kctx mutexes */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - js_kctx_info = &kctx->jctx.sched_info; - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - kbase_js_ctx_pullable(kctx, js, false)) - timer_sync = kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, js); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (timer_sync) - kbase_backend_ctx_count_changed(kbdev); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - /* Take lock before accessing list again */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } else { - bool timer_sync = false; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - list_for_each_entry_safe( - kctx, n, &kbdev->js_data.ctx_list_unpullable[js][prio], - jctx.sched_info.ctx.ctx_list_entry[js]) { - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - kbase_js_ctx_pullable(kctx, js, false)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, js); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (timer_sync) { - mutex_lock(&js_devdata->runpool_mutex); - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&js_devdata->runpool_mutex); - } - } - } - } - mutex_unlock(&js_devdata->queue_mutex); - - /* Restart atom processing */ - kbase_js_sched_all(kbdev); - - /* JS Resume complete */ -} - -bool kbase_js_is_atom_valid(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - CSTD_UNUSED(kbdev); - - if ((katom->core_req & BASE_JD_REQ_FS) && - (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T))) - return false; - - if ((katom->core_req & BASE_JD_REQ_JOB_SLOT) && (katom->jobslot >= BASE_JM_MAX_NR_SLOTS)) - return false; - - return true; -} - -static unsigned int kbase_js_get_slot(struct kbase_device *kbdev, struct kbase_jd_atom *katom) -{ - if (katom->core_req & BASE_JD_REQ_JOB_SLOT) - return katom->jobslot; - - if (katom->core_req & BASE_JD_REQ_FS) - return 0; - - if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { - if (katom->device_nr == 1 && kbdev->gpu_props.num_core_groups == 2) - return 2; - } - - return 1; -} - -bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - bool enqueue_required; - - katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - lockdep_assert_held(&kctx->jctx.lock); - - /* If slot will transition from unpullable to pullable then add to - * pullable list - */ - enqueue_required = jsctx_rb_none_to_pull(kctx, katom->slot_nr); - - if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || - (katom->pre_dep && - (katom->pre_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { - int prio = katom->sched_priority; - unsigned int js = katom->slot_nr; - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; - - dev_dbg(kctx->kbdev->dev, "Add atom %pK to X_DEP list (s:%u)\n", (void *)katom, js); - - list_add_tail(&katom->queue, &queue->x_dep_head); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - enqueue_required = false; - } else { - dev_dbg(kctx->kbdev->dev, "Atom %pK not added to X_DEP list\n", (void *)katom); - /* Check if there are lower priority jobs to soft stop */ - kbase_job_slot_ctx_priority_check_locked(kctx, katom); - - /* Add atom to ring buffer. */ - jsctx_tree_add(kctx, katom); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; - } - - dev_dbg(kctx->kbdev->dev, "Enqueue of kctx %pK is %srequired to submit atom %pK\n", kctx, - enqueue_required ? "" : "not ", katom); - - return enqueue_required; -} - -/** - * kbase_js_move_to_tree - Move atom (and any dependent atoms) to the - * runnable_tree, ready for execution - * @katom: Atom to submit - * - * It is assumed that @katom does not have KBASE_KATOM_FLAG_X_DEP_BLOCKED set, - * but is still present in the x_dep list. If @katom has a same-slot dependent - * atom then that atom (and any dependents) will also be moved. - */ -static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&katom->kctx->kbdev->hwaccess_lock); - - while (katom) { - WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); - - if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { - dev_dbg(katom->kctx->kbdev->dev, - "Del atom %pK from X_DEP list in js_move_to_tree\n", (void *)katom); - - list_del(&katom->queue); - katom->atom_flags &= ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - jsctx_tree_add(katom->kctx, katom); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; - } else { - dev_dbg(katom->kctx->kbdev->dev, - "Atom %pK blocked on x-dep in js_move_to_tree\n", (void *)katom); - break; - } - - katom = katom->post_dep; - } -} - -/** - * kbase_js_evict_deps - Evict dependencies of a failed atom. - * @kctx: Context pointer - * @katom: Pointer to the atom that has failed. - * @js: The job slot the katom was run on. - * @prio: Priority of the katom. - * - * Remove all post dependencies of an atom from the context ringbuffers. - * - * The original atom's event_code will be propagated to all dependent atoms. - * - * Context: Caller must hold the HW access lock - */ -static void kbase_js_evict_deps(struct kbase_context *kctx, struct kbase_jd_atom *katom, - unsigned int js, int prio) -{ - struct kbase_jd_atom *x_dep = katom->x_post_dep; - struct kbase_jd_atom *next_katom = katom->post_dep; - - CSTD_UNUSED(js); - CSTD_UNUSED(prio); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - if (next_katom) { - KBASE_DEBUG_ASSERT(next_katom->status != KBASE_JD_ATOM_STATE_HW_COMPLETED); - next_katom->will_fail_event_code = katom->event_code; - } - - /* Has cross slot depenency. */ - if (x_dep && (x_dep->atom_flags & - (KBASE_KATOM_FLAG_JSCTX_IN_TREE | KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { - /* Remove dependency.*/ - x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; - - dev_dbg(kctx->kbdev->dev, "Cleared X_DEP flag on atom %pK\n", (void *)x_dep); - - /* Fail if it had a data dependency. */ - if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) - x_dep->will_fail_event_code = katom->event_code; - - if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) - kbase_js_move_to_tree(x_dep); - } -} - -struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js) -{ - struct kbase_jd_atom *katom; - struct kbasep_js_device_data *js_devdata; - struct kbase_device *kbdev; - int pulled; - - KBASE_DEBUG_ASSERT(kctx); - - kbdev = kctx->kbdev; - dev_dbg(kbdev->dev, "JS: pulling an atom from kctx %pK (s:%u)\n", (void *)kctx, js); - - js_devdata = &kbdev->js_data; - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) { - dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %pK\n", (void *)kctx); - return NULL; - } - if (kbase_pm_is_suspending(kbdev) || kbase_io_is_aw_removed(kbdev)) - return NULL; - - katom = jsctx_rb_peek(kctx, js); - if (!katom) { - dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %pK (s:%u)\n", (void *)kctx, js); - return NULL; - } - if (kbase_jsctx_slot_prio_is_blocked(kctx, js, katom->sched_priority)) { - dev_dbg(kbdev->dev, - "JS: kctx %pK is blocked from submitting atoms at priority %d and lower (s:%u)\n", - (void *)kctx, katom->sched_priority, js); - return NULL; - } - if (atomic_read(&katom->blocked)) { - dev_dbg(kbdev->dev, "JS: Atom %pK is blocked in js_pull\n", (void *)katom); - return NULL; - } - - /* Due to ordering restrictions when unpulling atoms on failure, we do - * not allow multiple runs of fail-dep atoms from the same context to be - * present on the same slot - */ - if (katom->pre_dep && kbase_jsctx_slot_atoms_pulled(kctx, js)) { - struct kbase_jd_atom *prev_atom = kbase_backend_inspect_tail(kbdev, js); - - if (prev_atom && prev_atom->kctx != kctx) - return NULL; - } - - if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { - if (katom->x_pre_dep->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || - katom->x_pre_dep->will_fail_event_code) { - dev_dbg(kbdev->dev, - "JS: X pre-dep %pK is not present in slot FIFO or will fail\n", - (void *)katom->x_pre_dep); - return NULL; - } - if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && - kbase_backend_nr_atoms_on_slot(kbdev, js)) { - dev_dbg(kbdev->dev, - "JS: Atom %pK has cross-slot fail dependency and atoms on slot (s:%u)\n", - (void *)katom, js); - return NULL; - } - } - - KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JS_PULL_JOB, kctx, katom, katom->jc, js, - (u64)katom->sched_priority); - kbase_ctx_flag_set(kctx, KCTX_PULLED); - kbase_ctx_flag_set(kctx, (KCTX_PULLED_SINCE_ACTIVE_JS0 << js)); - - pulled = kbase_jsctx_slot_atom_pulled_inc(kctx, katom); - if (pulled == 1 && !kctx->slots_pullable) { - WARN_ON(kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_set(kctx, KCTX_RUNNABLE_REF); - atomic_inc(&kbdev->js_data.nr_contexts_runnable); - } - jsctx_rb_pull(kctx, katom); - - kbase_ctx_sched_retain_ctx_refcount(kctx); - - katom->ticks = 0; - - dev_dbg(kbdev->dev, "JS: successfully pulled atom %pK from kctx %pK (s:%u)\n", - (void *)katom, (void *)kctx, js); - - return katom; -} - -static void js_return_worker(struct work_struct *data) -{ - struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); - struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; - struct kbasep_js_atom_retained_state retained_state; - unsigned int js = katom->slot_nr; - bool slot_became_unblocked; - bool timer_sync = false; - bool context_idle = false; - unsigned long flags; - base_jd_core_req core_req = katom->core_req; - u64 cache_jc = katom->jc; - - dev_dbg(kbdev->dev, "%s for atom %pK with event code 0x%x\n", __func__, (void *)katom, - katom->event_code); - - KBASE_KTRACE_ADD_JM(kbdev, JS_RETURN_WORKER, kctx, katom, katom->jc, 0); - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(kbdev, katom); - - kbase_backend_complete_wq(kbdev, katom); - - kbasep_js_atom_retained_state_copy(&retained_state, katom); - - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - - atomic_dec(&katom->blocked); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - slot_became_unblocked = kbase_jsctx_slot_atom_pulled_dec(kctx, katom); - - if (!kbase_jsctx_slot_atoms_pulled(kctx, js) && jsctx_rb_none_to_pull(kctx, js)) - timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); - - /* If the context is now unblocked on this slot after soft-stopped - * atoms, then only mark it as pullable on this slot if it is not - * idle - */ - if (slot_became_unblocked && kbase_jsctx_atoms_pulled(kctx) && - kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, js); - - if (!kbase_jsctx_atoms_pulled(kctx)) { - dev_dbg(kbdev->dev, "No atoms currently pulled from context %pK\n", (void *)kctx); - - if (!kctx->slots_pullable) { - dev_dbg(kbdev->dev, "Context %pK %s counted as runnable\n", (void *)kctx, - kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF) ? "is" : "isn't"); - - WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); - atomic_dec(&kbdev->js_data.nr_contexts_runnable); - timer_sync = true; - } - - if (kctx->as_nr != KBASEP_AS_NR_INVALID && !kbase_ctx_flag(kctx, KCTX_DYING)) { - unsigned int num_slots = kbdev->gpu_props.num_job_slots; - unsigned int slot; - - if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) - kbasep_js_set_submit_allowed(js_devdata, kctx); - - for (slot = 0; slot < num_slots; slot++) { - if (kbase_js_ctx_pullable(kctx, slot, true)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, slot); - } - } - - kbase_jm_idle_ctx(kbdev, kctx); - - context_idle = true; - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (context_idle) { - dev_dbg(kbdev->dev, "Context %pK %s counted as active\n", (void *)kctx, - kbase_ctx_flag(kctx, KCTX_ACTIVE) ? "is" : "isn't"); - WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); - kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); - kbase_pm_context_idle(kbdev); - } - - if (timer_sync) - kbase_js_sync_timers(kbdev); - - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - - dev_dbg(kbdev->dev, "JS: retained state %s finished", - kbasep_js_has_atom_finished(&retained_state) ? "has" : "hasn't"); - - WARN_ON(kbasep_js_has_atom_finished(&retained_state)); - - kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &retained_state); - - kbase_js_sched_all(kbdev); - - kbase_backend_complete_wq_post_sched(kbdev, core_req); - - KBASE_KTRACE_ADD_JM(kbdev, JS_RETURN_WORKER_END, kctx, NULL, cache_jc, 0); - - dev_dbg(kbdev->dev, "Leaving %s for atom %pK\n", __func__, (void *)katom); -} - -void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - dev_dbg(kctx->kbdev->dev, "Unpulling atom %pK in kctx %pK\n", (void *)katom, (void *)kctx); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - jsctx_rb_unpull(kctx, katom); - - WARN_ON(work_pending(&katom->work)); - - /* Block re-submission until workqueue has run */ - atomic_inc(&katom->blocked); - - kbase_job_check_leave_disjoint(kctx->kbdev, katom); - - INIT_WORK(&katom->work, js_return_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); -} - -bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - struct kbasep_js_kctx_info *js_kctx_info; - struct kbasep_js_device_data *js_devdata; - struct kbase_device *kbdev; - unsigned long flags; - bool timer_sync = false; - unsigned int atom_slot; - bool context_idle = false; - int prio = katom->sched_priority; - - kbdev = kctx->kbdev; - atom_slot = katom->slot_nr; - - dev_dbg(kbdev->dev, "%s for atom %pK (s:%u)\n", __func__, (void *)katom, atom_slot); - - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - - mutex_lock(&js_devdata->runpool_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { - bool slot_became_unblocked; - - dev_dbg(kbdev->dev, "Atom %pK is in runnable_tree\n", (void *)katom); - - slot_became_unblocked = kbase_jsctx_slot_atom_pulled_dec(kctx, katom); - context_idle = !kbase_jsctx_atoms_pulled(kctx); - - if (!kbase_jsctx_atoms_pulled(kctx) && !kctx->slots_pullable) { - WARN_ON(!kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF)); - kbase_ctx_flag_clear(kctx, KCTX_RUNNABLE_REF); - atomic_dec(&kbdev->js_data.nr_contexts_runnable); - timer_sync = true; - } - - /* If this slot has been blocked due to soft-stopped atoms, and - * all atoms have now been processed at this priority level and - * higher, then unblock the slot - */ - if (slot_became_unblocked) { - dev_dbg(kbdev->dev, - "kctx %pK is no longer blocked from submitting on slot %u at priority %d or higher\n", - (void *)kctx, atom_slot, prio); - - if (kbase_js_ctx_pullable(kctx, atom_slot, true)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, - atom_slot); - } - } - WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); - - if (!kbase_jsctx_slot_atoms_pulled(kctx, atom_slot) && - jsctx_rb_none_to_pull(kctx, atom_slot)) { - if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot])) - timer_sync |= kbase_js_ctx_list_remove_nolock(kctx->kbdev, kctx, atom_slot); - } - - /* - * If submission is disabled on this context (most likely due to an - * atom failure) and there are now no atoms left in the system then - * re-enable submission so that context can be scheduled again. - */ - if (!kbasep_js_is_submit_allowed(js_devdata, kctx) && !kbase_jsctx_atoms_pulled(kctx) && - !kbase_ctx_flag(kctx, KCTX_DYING)) { - unsigned int js; - - kbasep_js_set_submit_allowed(js_devdata, kctx); - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, js); - } - } else if (katom->x_post_dep && kbasep_js_is_submit_allowed(js_devdata, kctx)) { - unsigned int js; - - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, js); - } - } - - /* Mark context as inactive. The pm reference will be dropped later in - * jd_done_worker(). - */ - if (context_idle) { - dev_dbg(kbdev->dev, "kctx %pK is no longer active\n", (void *)kctx); - kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (timer_sync) - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&js_devdata->runpool_mutex); - - dev_dbg(kbdev->dev, "Leaving %s\n", __func__); - return context_idle; -} - -struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp) -{ - struct kbase_device *kbdev; - struct kbase_context *kctx = katom->kctx; - struct kbase_jd_atom *x_dep = katom->x_post_dep; - - kbdev = kctx->kbdev; - dev_dbg(kbdev->dev, "Atom %pK complete in kctx %pK (post-dep %pK)\n", (void *)katom, - (void *)kctx, (void *)x_dep); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - if (katom->will_fail_event_code) - katom->event_code = katom->will_fail_event_code; - - katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; - dev_dbg(kbdev->dev, "Atom %pK status to HW completed\n", (void *)katom); - if (kbase_is_quick_reset_enabled(kbdev)) { - kbdev->num_of_atoms_hw_completed++; - if (kbdev->num_of_atoms_hw_completed >= 20) - kbase_disable_quick_reset(kbdev); - } - - if (katom->event_code != BASE_JD_EVENT_DONE) { - kbase_js_evict_deps(kctx, katom, katom->slot_nr, katom->sched_priority); - } - - KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, NULL, katom->slot_nr, 0, TL_JS_EVENT_STOP); - - trace_sysgraph_gpu(SGR_COMPLETE, kctx->id, kbase_jd_atom_id(katom->kctx, katom), - katom->slot_nr); - - KBASE_TLSTREAM_TL_JD_DONE_START(kbdev, katom); - kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); - KBASE_TLSTREAM_TL_JD_DONE_END(kbdev, katom); - - /* Unblock cross dependency if present */ - if (x_dep && - (katom->event_code == BASE_JD_EVENT_DONE || - !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) && - (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)) { - bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, false); - x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; - dev_dbg(kbdev->dev, "Cleared X_DEP flag on atom %pK\n", (void *)x_dep); - - kbase_js_move_to_tree(x_dep); - - if (!was_pullable && kbase_js_ctx_pullable(kctx, x_dep->slot_nr, false)) - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, x_dep->slot_nr); - - if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { - dev_dbg(kbdev->dev, "Atom %pK is in runnable tree\n", (void *)x_dep); - return x_dep; - } - } else { - dev_dbg(kbdev->dev, "No cross-slot dep to unblock for atom %pK\n", (void *)katom); - } - - return NULL; -} - -void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask) -{ - struct kbasep_js_device_data *js_devdata; - struct kbase_context *last_active_kctx[BASE_JM_MAX_NR_SLOTS]; - bool timer_sync = false; - bool ctx_waiting[BASE_JM_MAX_NR_SLOTS]; - unsigned int js; - - KBASE_TLSTREAM_TL_JS_SCHED_START(kbdev, 0); - - dev_dbg(kbdev->dev, "%s kbdev %pK mask 0x%x\n", __func__, (void *)kbdev, - (unsigned int)js_mask); - - js_devdata = &kbdev->js_data; - - down(&js_devdata->schedule_sem); - mutex_lock(&js_devdata->queue_mutex); - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - last_active_kctx[js] = kbdev->hwaccess.active_kctx[js]; - ctx_waiting[js] = false; - } - - while (js_mask) { - js = (unsigned int)ffs((int)js_mask) - 1; - - while (1) { - struct kbase_context *kctx; - unsigned long flags; - bool context_idle = false; - - kctx = kbase_js_ctx_list_pop_head(kbdev, js); - - if (!kctx) { - js_mask &= ~(1UL << js); - dev_dbg(kbdev->dev, "No kctx on pullable list (s:%u)\n", js); - break; - } - - if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) { - context_idle = true; - - dev_dbg(kbdev->dev, "kctx %pK is not active (s:%u)\n", (void *)kctx, - js); - - if (kbase_pm_context_active_handle_suspend( - kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { - dev_dbg(kbdev->dev, "Suspend pending (s:%u)\n", js); - /* Suspend pending - return context to - * queue and stop scheduling - */ - mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - if (kbase_js_ctx_list_add_pullable_head(kctx->kbdev, kctx, - js)) - kbase_js_sync_timers(kbdev); - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - up(&js_devdata->schedule_sem); - KBASE_TLSTREAM_TL_JS_SCHED_END(kbdev, 0); - return; - } - kbase_ctx_flag_set(kctx, KCTX_ACTIVE); - } - - if (!kbase_js_use_ctx(kbdev, kctx, js)) { - mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - - dev_dbg(kbdev->dev, "kctx %pK cannot be used at this time\n", kctx); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (kbase_js_ctx_pullable(kctx, js, false) || - kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) - timer_sync |= kbase_js_ctx_list_add_pullable_head_nolock( - kctx->kbdev, kctx, js); - else - timer_sync |= kbase_js_ctx_list_add_unpullable_nolock( - kctx->kbdev, kctx, js); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - if (context_idle) { - WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); - kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); - kbase_pm_context_idle(kbdev); - } - - /* No more jobs can be submitted on this slot */ - js_mask &= ~(1UL << js); - break; - } - mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - kbase_ctx_flag_clear(kctx, KCTX_PULLED); - - if (!kbase_jm_kick(kbdev, 1UL << js)) { - dev_dbg(kbdev->dev, "No more jobs can be submitted (s:%u)\n", js); - js_mask &= ~(1UL << js); - } - if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { - bool pullable; - - dev_dbg(kbdev->dev, "No atoms pulled from kctx %pK (s:%u)\n", - (void *)kctx, js); - - pullable = kbase_js_ctx_pullable(kctx, js, true); - - /* Failed to pull jobs - push to head of list. - * Unless this context is already 'active', in - * which case it's effectively already scheduled - * so push it to the back of the list. - */ - if (pullable && kctx == last_active_kctx[js] && - kbase_ctx_flag(kctx, (KCTX_PULLED_SINCE_ACTIVE_JS0 << js))) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock( - kctx->kbdev, kctx, js); - else if (pullable) - timer_sync |= kbase_js_ctx_list_add_pullable_head_nolock( - kctx->kbdev, kctx, js); - else - timer_sync |= kbase_js_ctx_list_add_unpullable_nolock( - kctx->kbdev, kctx, js); - - /* If this context is not the active context, - * but the active context is pullable on this - * slot, then we need to remove the active - * marker to prevent it from submitting atoms in - * the IRQ handler, which would prevent this - * context from making progress. - */ - if (last_active_kctx[js] && kctx != last_active_kctx[js] && - kbase_js_ctx_pullable(last_active_kctx[js], js, true)) - ctx_waiting[js] = true; - - if (context_idle) { - kbase_jm_idle_ctx(kbdev, kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - WARN_ON(!kbase_ctx_flag(kctx, KCTX_ACTIVE)); - kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); - kbase_pm_context_idle(kbdev); - } else { - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - - js_mask &= ~(1UL << js); - break; /* Could not run atoms on this slot */ - } - - dev_dbg(kbdev->dev, "Push kctx %pK to back of list\n", (void *)kctx); - if (kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock(kctx->kbdev, - kctx, js); - else - timer_sync |= kbase_js_ctx_list_add_unpullable_nolock(kctx->kbdev, - kctx, js); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - } - } - - if (timer_sync) - kbase_js_sync_timers(kbdev); - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - if (kbdev->hwaccess.active_kctx[js] == last_active_kctx[js] && ctx_waiting[js]) { - dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%u)\n", - (void *)last_active_kctx[js], js); - kbdev->hwaccess.active_kctx[js] = NULL; - } - } - - mutex_unlock(&js_devdata->queue_mutex); - up(&js_devdata->schedule_sem); - KBASE_TLSTREAM_TL_JS_SCHED_END(kbdev, 0); -} - -void kbase_js_zap_context(struct kbase_context *kctx) -{ - struct kbase_device *kbdev = kctx->kbdev; - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; - - /* - * Critical assumption: No more submission is possible outside of the - * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs) - * whilst the struct kbase_context is terminating. - */ - - /* First, atomically do the following: - * - mark the context as dying - * - try to evict it from the queue - */ - mutex_lock(&kctx->jctx.lock); - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - kbase_ctx_flag_set(kctx, KCTX_DYING); - - dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %pK", kctx); - - /* - * At this point we know: - * - If eviction succeeded, it was in the queue, but now no - * longer is - * - We must cancel the jobs here. No Power Manager active reference to - * release. - * - This happens asynchronously - kbase_jd_zap_context() will wait for - * those jobs to be killed. - * - If eviction failed, then it wasn't in the queue. It is one - * of the following: - * - a. it didn't have any jobs, and so is not in the Queue or - * the Run Pool (not scheduled) - * - Hence, no more work required to cancel jobs. No Power Manager - * active reference to release. - * - b. it was in the middle of a scheduling transaction (and thus must - * have at least 1 job). This can happen from a syscall or a - * kernel thread. We still hold the jsctx_mutex, and so the thread - * must be waiting inside kbasep_js_try_schedule_head_ctx(), - * before checking whether the runpool is full. That thread will - * continue after we drop the mutex, and will notice the context - * is dying. It will rollback the transaction, killing all jobs at - * the same time. kbase_jd_zap_context() will wait for those jobs - * to be killed. - * - Hence, no more work required to cancel jobs, or to release the - * Power Manager active reference. - * - c. it is scheduled, and may or may not be running jobs - * - We must cause it to leave the runpool by stopping it from - * submitting any more jobs. When it finally does leave, - * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs - * (because it is dying), release the Power Manager active reference, - * and will not requeue the context in the queue. - * kbase_jd_zap_context() will wait for those jobs to be killed. - * - Hence, work required just to make it leave the runpool. Cancelling - * jobs and releasing the Power manager active reference will be - * handled when it leaves the runpool. - */ - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED)) { - unsigned long flags; - unsigned int js; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* The following events require us to kill off remaining jobs - * and update PM book-keeping: - * - we evicted it correctly (it must have jobs to be in the - * Queue) - * - * These events need no action, but take this path anyway: - * - Case a: it didn't have any jobs, and was never in the Queue - * - Case b: scheduling transaction will be partially rolled- - * back (this already cancels the jobs) - */ - - KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, - kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - dev_dbg(kbdev->dev, "Zap: Ctx %pK scheduled=0", kctx); - - /* Only cancel jobs when we evicted from the - * queue. No Power Manager active reference was held. - * - * Having is_dying set ensures that this kills, and doesn't - * requeue - */ - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); - - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - mutex_unlock(&kctx->jctx.lock); - } else { - unsigned long flags; - bool was_retained; - CSTD_UNUSED(was_retained); - - /* Case c: didn't evict, but it is scheduled - it's in the Run - * Pool - */ - KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, - kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - dev_dbg(kbdev->dev, "Zap: Ctx %pK is in RunPool", kctx); - - /* Disable the ctx from submitting any more jobs */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - kbasep_js_clear_submit_allowed(js_devdata, kctx); - - /* Retain and (later) release the context whilst it is now - * disallowed from submitting jobs - ensures that someone - * somewhere will be removing the context later on - */ - was_retained = kbase_ctx_sched_inc_refcount_nolock(kctx); - - /* Since it's scheduled and we have the jsctx_mutex, it must be - * retained successfully - */ - KBASE_DEBUG_ASSERT(was_retained); - - dev_dbg(kbdev->dev, "Zap: Ctx %pK Kill Any Running jobs", kctx); - - /* Cancel any remaining running jobs for this kctx - if any. - * Submit is disallowed which takes effect immediately, so no - * more new jobs will appear after we do this. - */ - kbase_backend_jm_kill_running_jobs_from_kctx(kctx); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - mutex_unlock(&kctx->jctx.lock); - - dev_dbg(kbdev->dev, - "Zap: Ctx %pK Release (may or may not schedule out immediately)", kctx); - - kbasep_js_runpool_release_ctx(kbdev, kctx); - } - - KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u); - - /* After this, you must wait on both the - * kbase_jd_context::zero_jobs_wait and the - * kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the jobs - * to be destroyed, and the context to be de-scheduled (if it was on the - * runpool). - * - * kbase_jd_zap_context() will do this. - */ -} - -static inline int trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - CSTD_UNUSED(kbdev); - return atomic_read(&kctx->refcount); -} - -/** - * kbase_js_foreach_ctx_job(): - Call a function on all jobs in context - * @kctx: Pointer to context. - * @callback: Pointer to function to call for each job. - * - * Call a function on all jobs belonging to a non-queued, non-running - * context, and detach the jobs from the context as it goes. - * - * Due to the locks that might be held at the time of the call, the callback - * may need to defer work on a workqueue to complete its actions (e.g. when - * cancelling jobs) - * - * Atoms will be removed from the queue, so this must only be called when - * cancelling jobs (which occurs as part of context destruction). - * - * The locking conditions on the caller are as follows: - * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex. - */ -static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, kbasep_js_ctx_job_cb *callback) -{ - struct kbase_device *kbdev; - unsigned long flags; - unsigned int js; - - kbdev = kctx->kbdev; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, 0u, - trace_get_refcnt(kbdev, kctx)); - - /* Invoke callback on jobs on each slot in turn */ - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) - jsctx_queue_foreach(kctx, js, callback); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -base_jd_prio kbase_js_priority_check(struct kbase_device *kbdev, base_jd_prio priority) -{ - struct priority_control_manager_device *pcm_device = kbdev->pcm_dev; - int req_priority, out_priority; - - req_priority = kbasep_js_atom_prio_to_sched_prio(priority); - out_priority = req_priority; - /* Does not use pcm defined priority check if PCM not defined or if - * kbasep_js_atom_prio_to_sched_prio returns an error - * (KBASE_JS_ATOM_SCHED_PRIO_INVALID). - */ - if (pcm_device && (req_priority != KBASE_JS_ATOM_SCHED_PRIO_INVALID)) - out_priority = pcm_device->ops.pcm_scheduler_priority_check(pcm_device, current, - req_priority); - return kbasep_js_sched_prio_to_atom_prio(kbdev, out_priority); -} diff --git a/drivers/gpu/arm/valhall/mali_kbase_js.h b/drivers/gpu/arm/valhall/mali_kbase_js.h deleted file mode 100644 index 89c3b45c735b..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_js.h +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Job Scheduler APIs. - */ - -#ifndef _KBASE_JS_H_ -#define _KBASE_JS_H_ - -#include "context/mali_kbase_context.h" -#include "mali_kbase_defs.h" -#include "mali_kbase_debug.h" -#include -#include "jm/mali_kbase_jm_js.h" -#include "jm/mali_kbase_js_defs.h" - -#endif /* _KBASE_JS_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.c b/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.c deleted file mode 100644 index 6fc6b8a93027..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.c +++ /dev/null @@ -1,324 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include - -/* - * Private functions follow - */ - -/** - * kbasep_js_ctx_attr_runpool_retain_attr - Check whether a ctx has a certain attribute - * and if so, retain that attribute on the runpool. - * - * @kbdev: Device pointer - * @kctx: KBase context - * @attribute: Atribute to check/retain - * - * Requires: - * - jsctx mutex - * - runpool_irq spinlock - * - ctx is scheduled on the runpool - * - * Return: true indicates a change in ctx attributes state of the runpool. - * In this state, the scheduler might be able to submit more jobs than - * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() - * or similar is called sometime later. - * false indicates no change in ctx attributes state of the runpool. - */ -static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, - struct kbase_context *kctx, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - bool runpool_state_changed = false; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - - KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { - KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX); - ++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); - - if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) { - /* First refcount indicates a state change */ - runpool_state_changed = true; - KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, - attribute); - } - } - - return runpool_state_changed; -} - -/** - * kbasep_js_ctx_attr_runpool_release_attr - Check whether a ctx has a certain attribute, - * and if so, release that attribute on the runpool. - * - * @kbdev: Device pointer - * @kctx: KBase context - * @attribute: Atribute to release - * - * Requires: - * - jsctx mutex - * - runpool_irq spinlock - * - ctx is scheduled on the runpool - * - * Return: true indicates a change in ctx attributes state of the runpool. - * In this state, the scheduler might be able to submit more jobs than - * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() - * or similar is called sometime later. - * false indicates no change in ctx attributes state of the runpool. - */ -static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, - struct kbase_context *kctx, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_device_data *js_devdata; - struct kbasep_js_kctx_info *js_kctx_info; - bool runpool_state_changed = false; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - - if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) { - KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0); - --(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]); - - if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) { - /* Last de-refcount indicates a state change */ - runpool_state_changed = true; - KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, - attribute); - } - } - - return runpool_state_changed; -} - -/** - * kbasep_js_ctx_attr_ctx_retain_attr - Retain a certain attribute on a ctx, - * also retaining it on the runpool if the context is scheduled. - * - * @kbdev: Device pointer - * @kctx: KBase context - * @attribute: Atribute to retain - * - * Requires: - * - jsctx mutex - * - If the context is scheduled, then runpool_irq spinlock must also be held - * - * Return: true indicates a change in ctx attributes state of the runpool. - * This may allow the scheduler to submit more jobs than previously. - * false indicates no change in ctx attributes state of the runpool. - */ -static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, - struct kbase_context *kctx, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_kctx_info *js_kctx_info; - bool runpool_state_changed = false; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&kbdev->hwaccess_lock); - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX); - - ++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); - - if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { - /* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */ - KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute); - runpool_state_changed = - kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute); - } - - return runpool_state_changed; -} - -/** - * kbasep_js_ctx_attr_ctx_release_attr - Release a certain attribute on a ctx, - * also releasing it from the runpool if the context is scheduled. - * - * @kbdev: Device pointer - * @kctx: KBase context - * @attribute: Atribute to release - * - * Requires: - * - jsctx mutex - * - If the context is scheduled, then runpool_irq spinlock must also be held - * - * Return: true indicates a change in ctx attributes state of the runpool. - * This may allow the scheduler to submit more jobs than previously. - * false indicates no change in ctx attributes state of the runpool. - */ -static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, - struct kbase_context *kctx, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_kctx_info *js_kctx_info; - bool runpool_state_changed = false; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_kctx_info = &kctx->jctx.sched_info; - - lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex); - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0); - - if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) { - lockdep_assert_held(&kbdev->hwaccess_lock); - /* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */ - runpool_state_changed = - kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute); - KBASE_KTRACE_ADD_JM(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute); - } - - /* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */ - --(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); - - return runpool_state_changed; -} - -/* - * More commonly used public functions - */ - -void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - bool runpool_state_changed; - int i; - - /* Retain any existing attributes */ - for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { - if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr)i) != false) { - /* The context is being scheduled in, so update the runpool with the new attributes */ - runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr( - kbdev, kctx, (enum kbasep_js_ctx_attr)i); - - /* We don't need to know about state changed, because retaining a - * context occurs on scheduling it, and that itself will also try - * to run new atoms - */ - CSTD_UNUSED(runpool_state_changed); - } - } -} - -bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) -{ - bool runpool_state_changed = false; - int i; - - /* Release any existing attributes */ - for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) { - if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr)i) != false) { - /* The context is being scheduled out, so update the runpool on the removed attributes */ - runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr( - kbdev, kctx, (enum kbasep_js_ctx_attr)i); - } - } - - return runpool_state_changed; -} - -void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *katom) -{ - bool runpool_state_changed = false; - base_jd_core_req core_req; - - KBASE_DEBUG_ASSERT(katom); - core_req = katom->core_req; - - if (core_req & BASE_JD_REQ_ONLY_COMPUTE) - runpool_state_changed |= - kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); - else - runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr( - kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); - - if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && - (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { - /* Atom that can run on slot1 or slot2, and can use all cores */ - runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr( - kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); - } - - /* We don't need to know about state changed, because retaining an atom - * occurs on adding it, and that itself will also try to run new atoms - */ - CSTD_UNUSED(runpool_state_changed); -} - -bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state) -{ - bool runpool_state_changed = false; - base_jd_core_req core_req; - - KBASE_DEBUG_ASSERT(katom_retained_state); - core_req = katom_retained_state->core_req; - - /* No-op for invalid atoms */ - if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false) - return false; - - if (core_req & BASE_JD_REQ_ONLY_COMPUTE) - runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr( - kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE); - else - runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr( - kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE); - - if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && - (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) { - /* Atom that can run on slot1 or slot2, and can use all cores */ - runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr( - kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); - } - - return runpool_state_changed; -} diff --git a/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.h b/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.h deleted file mode 100644 index 471f54081d75..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_js_ctx_attr.h +++ /dev/null @@ -1,152 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Job Scheduler Context Attribute APIs - */ - -#ifndef _KBASE_JS_CTX_ATTR_H_ -#define _KBASE_JS_CTX_ATTR_H_ - -/** - * kbasep_js_ctx_attr_runpool_retain_ctx - Retain all attributes of a context - * - * @kbdev: KBase device - * @kctx: KBase context - * - * This occurs on scheduling in the context on the runpool (but after - * is_scheduled is set) - * - * Requires: - * - jsctx mutex - * - runpool_irq spinlock - * - ctx->is_scheduled is true - */ -void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbasep_js_ctx_attr_runpool_release_ctx - Release all attributes of a context - * - * @kbdev: KBase device - * @kctx: KBase context - * - * This occurs on scheduling out the context from the runpool (but before - * is_scheduled is cleared) - * - * Requires: - * - jsctx mutex - * - runpool_irq spinlock - * - ctx->is_scheduled is true - * - * Return: true indicates a change in ctx attributes state of the runpool. - * In this state, the scheduler might be able to submit more jobs than - * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() - * or similar is called sometime later. - * false indicates no change in ctx attributes state of the runpool. - */ -bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx); - -/** - * kbasep_js_ctx_attr_ctx_retain_atom - Retain all attributes of an atom - * - * @kbdev: KBase device - * @kctx: KBase context - * @katom: Atom - * - * This occurs on adding an atom to a context - * - * Requires: - * - jsctx mutex - * - If the context is scheduled, then runpool_irq spinlock must also be held - */ -void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_jd_atom *katom); - -/** - * kbasep_js_ctx_attr_ctx_release_atom - Release all attributes of an atom, - * given its retained state. - * - * @kbdev: KBase device - * @kctx: KBase context - * @katom_retained_state: Retained state - * - * This occurs after (permanently) removing an atom from a context - * - * Requires: - * - jsctx mutex - * - If the context is scheduled, then runpool_irq spinlock must also be held - * - * This is a no-op when \a katom_retained_state is invalid. - * - * Return: true indicates a change in ctx attributes state of the runpool. - * In this state, the scheduler might be able to submit more jobs than - * previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock() - * or similar is called sometime later. - * false indicates no change in ctx attributes state of the runpool. - */ -bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbasep_js_atom_retained_state *katom_retained_state); - -/* - * Requires: - * - runpool_irq spinlock - */ -static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_device_data *js_devdata; - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_devdata = &kbdev->js_data; - - return js_devdata->runpool_irq.ctx_attr_ref_count[attribute]; -} - -/* - * Requires: - * - runpool_irq spinlock - */ -static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kbdev, - enum kbasep_js_ctx_attr attribute) -{ - /* In general, attributes are 'on' when they have a non-zero refcount (note: the refcount will never be < 0) */ - return (bool)kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute); -} - -/* - * Requires: - * - jsctx mutex - */ -static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, - enum kbasep_js_ctx_attr attribute) -{ - struct kbasep_js_kctx_info *js_kctx_info; - - KBASE_DEBUG_ASSERT(kctx != NULL); - KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT); - js_kctx_info = &kctx->jctx.sched_info; - - /* In general, attributes are 'on' when they have a refcount (which should never be < 0) */ - return (bool)(js_kctx_info->ctx.ctx_attr_ref_count[attribute]); -} - -#endif /* _KBASE_JS_DEFS_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.c b/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.c deleted file mode 100644 index 13e225507017..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.c +++ /dev/null @@ -1,880 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * mali_kbase_kinstr_jm.c - * Kernel driver public interface to job manager atom tracing - */ - -#include "mali_kbase_kinstr_jm.h" -#include - -#include "mali_kbase.h" -#include "mali_kbase_linux.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Explicitly include epoll header for old kernels. Not required from 4.16. */ -#if KERNEL_VERSION(4, 16, 0) > LINUX_VERSION_CODE -#include -#endif - -/* Define static_assert(). - * - * The macro was introduced in kernel 5.1. But older vendor kernels may define - * it too. - */ -#if KERNEL_VERSION(5, 1, 0) <= LINUX_VERSION_CODE -#include -#elif !defined(static_assert) -// Stringify the expression if no message is given. -#define static_assert(e, ...) __static_assert(e, #__VA_ARGS__, #e) -#define __static_assert(e, msg, ...) _Static_assert(e, msg) -#endif - -/* The module printing prefix */ -#define PR_ "mali_kbase_kinstr_jm: " - -/* Allows us to perform ASM goto for the tracing - * https://www.kernel.org/doc/Documentation/static-keys.txt - */ -DEFINE_STATIC_KEY_FALSE(basep_kinstr_jm_reader_static_key); - -#define KBASE_KINSTR_JM_VERSION 2 - -/** - * struct kbase_kinstr_jm - The context for the kernel job manager atom tracing - * @readers: a bitlocked list of opened readers. Readers are attached to the - * private data of a file descriptor that the user opens with the - * KBASE_IOCTL_KINSTR_JM_FD IO control call. - * @refcount: reference count for the context. Any reader will have a link - * back to the context so that they can remove themselves from the - * list. - * - * This is opaque outside this compilation unit - */ -struct kbase_kinstr_jm { - struct hlist_bl_head readers; - struct kref refcount; -}; - -/** - * struct kbase_kinstr_jm_atom_state_change - Represents an atom changing to a - * new state - * @timestamp: Raw monotonic nanoseconds of the state change - * @state: The state that the atom has moved to - * @atom: The atom number that has changed state - * @flags: Flags associated with the state change. See - * KBASE_KINSTR_JM_ATOM_STATE_FLAG_* defines. - * @reserved: Reserved for future use. - * @data: Extra data for the state change. Active member depends on state. - * @data.start: Extra data for the state change. Active member depends on - * state. - * @data.start.slot: Extra data for the state change. Active member depends on - * state. - * @data.padding: Padding - * - * We can add new fields to the structure and old user code will gracefully - * ignore the new fields. - * - * We can change the size of the structure and old user code will gracefully - * skip over the new size via `struct kbase_kinstr_jm_fd_out->size`. - * - * If we remove fields, the version field in `struct - * kbase_kinstr_jm_fd_out->version` will be incremented and old user code will - * gracefully fail and tell the user that the kernel API is too new and has - * backwards-incompatible changes. Note that one userspace can opt to handle - * multiple kernel major versions of the structure. - * - * If we need to change the _meaning_ of one of the fields, i.e. the state - * machine has had a incompatible change, we can keep the same members in the - * structure and update the version as above. User code will no longer - * recognise that it has the supported field and can gracefully explain to the - * user that the kernel API is no longer supported. - * - * When making changes to this structure, make sure they are either: - * - additions to the end (for minor version bumps (i.e. only a size increase)) - * such that the layout of existing fields doesn't change, or; - * - update the version reported to userspace so that it can fail explicitly. - */ -struct kbase_kinstr_jm_atom_state_change { - u64 timestamp; - s8 state; /* enum kbase_kinstr_jm_reader_atom_state */ - u8 atom; - u8 flags; - u8 reserved[1]; - /* Tagged union based on state. Ensure members are aligned correctly! */ - union { - struct { - u8 slot; - } start; - u8 padding[4]; - } data; -}; -static_assert(((1 << 8 * sizeof(((struct kbase_kinstr_jm_atom_state_change *)0)->state)) - 1) >= - KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT); - -#define KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW BIT(0) - -/** - * struct reader_changes - The circular buffer of kernel atom state changes - * @data: The allocated buffer. This is allocated when the user requests - * the reader file descriptor. It is released when the user calls - * close() on the fd. When accessing this, lock the producer spin - * lock to prevent races on the allocated memory. The consume lock - * does not need to be held because newly-inserted data will always - * be outside the currenly-read range. - * @producer: The producing spinlock which allows us to push changes into the - * buffer at the same time as a user read occurring. This needs to - * be locked when saving/restoring the IRQ because we can receive an - * interrupt from the GPU when an atom completes. The CPU could have - * a task preempted that is holding this lock. - * @consumer: The consuming mutex which locks around the user read(). - * Must be held when updating the tail of the circular buffer. - * @head: The head of the circular buffer. Can be used with Linux @c CIRC_ - * helpers. The producer should lock and update this with an SMP - * store when a new change lands. The consumer can read with an - * SMP load. This allows the producer to safely insert new changes - * into the circular buffer. - * @tail: The tail of the circular buffer. Can be used with Linux @c CIRC_ - * helpers. The producer should do a READ_ONCE load and the consumer - * should SMP store. - * @size: The number of changes that are allowed in @c data. Can be used - * with Linux @c CIRC_ helpers. Will always be a power of two. The - * producer lock should be held when updating this and stored with - * an SMP release memory barrier. This means that the consumer can - * do an SMP load. - * @threshold: The number of changes above which threads polling on the reader - * file descriptor will be woken up. - */ -struct reader_changes { - struct kbase_kinstr_jm_atom_state_change *data; - spinlock_t producer; - struct mutex consumer; - u32 head; - u32 tail; - u32 size; - u32 threshold; -}; - -/** - * reader_changes_is_valid_size() - Determines if requested changes buffer size - * is valid. - * @size: The requested memory size - * - * We have a constraint that the underlying physical buffer must be a - * power of two so that we can use the efficient circular buffer helpers that - * the kernel provides. It also needs to be representable within a u32. - * - * Return: - * * true - the size is valid - * * false - the size is invalid - */ -static inline bool reader_changes_is_valid_size(const size_t size) -{ - const size_t elem_size = sizeof(*((struct reader_changes *)0)->data); - const size_t size_size = sizeof(((struct reader_changes *)0)->size); - const size_t size_max = (1ull << (size_size * 8)) - 1; - - return is_power_of_2(size) && /* Is a power of two */ - ((size / elem_size) <= size_max); /* Small enough */ -} - -/** - * reader_changes_init() - Initializes the reader changes and allocates the - * changes buffer - * @changes: The context pointer, must point to a zero-inited allocated reader - * changes structure. We may support allocating the structure in the - * future. - * @size: The requested changes buffer size - * - * Return: - * (0, U16_MAX] - the number of data elements allocated - * -ERANGE - the requested memory size was invalid - * -ENOMEM - could not allocate the memory - */ -static int reader_changes_init(struct reader_changes *const changes, const size_t size) -{ - BUILD_BUG_ON((PAGE_SIZE % sizeof(*changes->data)) != 0); - - if (!reader_changes_is_valid_size(size)) { - pr_warn(PR_ "invalid size %zu\n", size); - return -ERANGE; - } - - changes->data = vmalloc(size); - if (!changes->data) - return -ENOMEM; - - spin_lock_init(&changes->producer); - mutex_init(&changes->consumer); - - changes->size = size / sizeof(*changes->data); - changes->threshold = - min(((size_t)(changes->size)) / 4, ((size_t)(PAGE_SIZE)) / sizeof(*changes->data)); - - return (int)changes->size; -} - -/** - * reader_changes_term() - Cleans up a reader changes structure - * @changes: The context to clean up - * - * Releases the allocated state changes memory - */ -static void reader_changes_term(struct reader_changes *const changes) -{ - struct kbase_kinstr_jm_atom_state_change *data = NULL; - unsigned long irq; - - /* - * Although changes->data is used on the consumer side, too, no active - * consumer is possible by the time we clean up the reader changes, so - * no need to take the consumer lock. However, we do need the producer - * lock because the list removal can race with list traversal. - */ - spin_lock_irqsave(&changes->producer, irq); - swap(changes->data, data); - spin_unlock_irqrestore(&changes->producer, irq); - - mutex_destroy(&changes->consumer); - vfree(data); -} - -/** - * reader_changes_count_locked() - Retrieves the count of state changes from the - * tail to the physical end of the buffer - * @changes: The state changes context - * - * The consumer mutex must be held. Uses the CIRC_CNT_TO_END macro to - * determine the count, so there may be more items. However, that's the maximum - * number that can be read in one contiguous read. - * - * Return: the number of changes in the circular buffer until the end of the - * allocation - */ -static u32 reader_changes_count_locked(struct reader_changes *const changes) -{ - u32 head; - - lockdep_assert_held_once(&changes->consumer); - - head = smp_load_acquire(&changes->head); - - return CIRC_CNT_TO_END(head, changes->tail, changes->size); -} - -/** - * reader_changes_count() - Retrieves the count of state changes from the - * tail to the physical end of the buffer - * @changes: The state changes context - * - * Return: the number of changes in the circular buffer until the end of the - * allocation - */ -static u32 reader_changes_count(struct reader_changes *const changes) -{ - u32 ret; - - mutex_lock(&changes->consumer); - ret = reader_changes_count_locked(changes); - mutex_unlock(&changes->consumer); - return ret; -} - -/** - * reader_changes_push() - Pushes a change into the reader circular buffer. - * @changes: The buffer to insert the change into - * @change: Kernel atom change to insert - * @wait_queue: The queue to be kicked when changes should be read from - * userspace. Kicked when a threshold is reached or there is - * overflow. - */ -static void reader_changes_push(struct reader_changes *const changes, - const struct kbase_kinstr_jm_atom_state_change *const change, - wait_queue_head_t *const wait_queue) -{ - u32 head, tail, size, space; - unsigned long irq; - struct kbase_kinstr_jm_atom_state_change *data; - - spin_lock_irqsave(&changes->producer, irq); - - /* We may be called for a reader_changes that's awaiting cleanup. */ - data = changes->data; - if (!data) - goto unlock; - - size = changes->size; - head = changes->head; - tail = smp_load_acquire(&changes->tail); - - space = CIRC_SPACE(head, tail, size); - if (space >= 1) { - data[head] = *change; - if (space == 1) { - data[head].flags |= KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW; - pr_warn(PR_ "overflow of circular buffer\n"); - } - smp_store_release(&changes->head, (head + 1) & (size - 1)); - } - - /* Wake for either overflow or over-threshold cases. */ - if (CIRC_CNT(head + 1, tail, size) >= changes->threshold) - wake_up_interruptible(wait_queue); - -unlock: - spin_unlock_irqrestore(&changes->producer, irq); -} - -/** - * struct reader - Allows the kernel state changes to be read by user space. - * @node: The node in the @c readers locked list - * @rcu_head: storage for the RCU callback to free this reader (see kfree_rcu) - * @changes: The circular buffer of user changes - * @wait_queue: A wait queue for poll - * @context: a pointer to the parent context that created this reader. Can be - * used to remove the reader from the list of readers. Reference - * counted. - * - * The reader is a circular buffer in kernel space. State changes are pushed - * into the buffer. The flow from user space is: - * - * * Request file descriptor with KBASE_IOCTL_KINSTR_JM_FD. This will - * allocate the kernel side circular buffer with a size specified in the - * ioctl argument. - * * The user will then poll the file descriptor for data - * * Upon receiving POLLIN, perform a read() on the file descriptor to get - * the data out. - * * The buffer memory will be freed when the file descriptor is closed - */ -struct reader { - struct hlist_bl_node node; - struct rcu_head rcu_head; - struct reader_changes changes; - wait_queue_head_t wait_queue; - struct kbase_kinstr_jm *context; -}; - -static struct kbase_kinstr_jm *kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx); -static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx); -static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, - struct reader *const reader); -static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, - struct reader *const reader); - -/** - * reader_term() - Terminate a instrumentation job manager reader context. - * @reader: Pointer to context to be terminated. - */ -static void reader_term(struct reader *const reader) -{ - if (!reader) - return; - - kbase_kinstr_jm_readers_del(reader->context, reader); - reader_changes_term(&reader->changes); - kbase_kinstr_jm_ref_put(reader->context); - - kfree_rcu(reader, rcu_head); -} - -/** - * reader_init() - Initialise a instrumentation job manager reader context. - * @out_reader: Non-NULL pointer to where the pointer to the created context - * will be stored on success. - * @ctx: the pointer to the parent context. Reference count will be - * increased if initialization is successful - * @num_changes: The number of changes to allocate a buffer for - * - * Return: 0 on success, else error code. - */ -static int reader_init(struct reader **const out_reader, struct kbase_kinstr_jm *const ctx, - size_t const num_changes) -{ - struct reader *reader = NULL; - const size_t change_size = sizeof(struct kbase_kinstr_jm_atom_state_change); - int status; - - if (!out_reader || !ctx || !num_changes) - return -EINVAL; - - reader = kzalloc(sizeof(*reader), GFP_KERNEL); - if (!reader) - return -ENOMEM; - - INIT_HLIST_BL_NODE(&reader->node); - init_waitqueue_head(&reader->wait_queue); - - reader->context = kbase_kinstr_jm_ref_get(ctx); - - status = reader_changes_init(&reader->changes, num_changes * change_size); - if (status < 0) - goto fail; - - status = kbase_kinstr_jm_readers_add(ctx, reader); - if (status < 0) - goto fail; - - *out_reader = reader; - - return 0; - -fail: - kbase_kinstr_jm_ref_put(reader->context); - kfree(reader); - return status; -} - -/** - * reader_release() - Invoked when the reader file descriptor is released - * @node: The inode that the file descriptor that the file corresponds to. In - * our case our reader file descriptor is backed by an anonymous node so - * not much is in this. - * @file: the file data. Our reader context is held in the private data - * Return: zero on success - */ -static int reader_release(struct inode *const node, struct file *const file) -{ - struct reader *const reader = file->private_data; - - CSTD_UNUSED(node); - - reader_term(reader); - file->private_data = NULL; - - return 0; -} - -/** - * reader_changes_copy_to_user() - Copy any changes from a changes structure to - * the user-provided buffer. - * @changes: The changes structure from which to copy. - * @buffer: The user buffer to copy the data to. - * @buffer_size: The number of bytes in the buffer. - * Return: The number of bytes copied or negative errno on failure. - */ -static ssize_t reader_changes_copy_to_user(struct reader_changes *const changes, - char __user *buffer, size_t buffer_size) -{ - ssize_t ret = 0; - struct kbase_kinstr_jm_atom_state_change const *src_buf = READ_ONCE(changes->data); - size_t const entry_size = sizeof(*src_buf); - size_t changes_tail, changes_count, read_size; - size_t copy_size; - - /* Needed for the quick buffer capacity calculation below. - * Note that we can't use is_power_of_2() since old compilers don't - * understand it's a constant expression. - */ -#define is_power_of_two(x) ((x) && !((x) & ((x)-1))) - static_assert(is_power_of_two(sizeof(struct kbase_kinstr_jm_atom_state_change))); -#undef is_power_of_two - - lockdep_assert_held_once(&changes->consumer); - - /* Read continuously until either: - * - we've filled the output buffer, or - * - there are no changes when we check. - * - * If more changes arrive while we're copying to the user, we can copy - * those as well, space permitting. - */ - do { - changes_tail = changes->tail; - changes_count = reader_changes_count_locked(changes); - - if (check_mul_overflow(changes_count, entry_size, ©_size)) { - ret = -EINVAL; - goto exit; - } - - read_size = min(copy_size, buffer_size & ~(entry_size - 1)); - - if (!read_size) - break; - - if (copy_to_user(buffer, &(src_buf[changes_tail]), read_size)) { - ret = -EFAULT; - goto exit; - } - - buffer += read_size; - buffer_size -= read_size; - ret += (ssize_t)read_size; - changes_tail = (changes_tail + read_size / entry_size) & (changes->size - 1); - smp_store_release(&changes->tail, changes_tail); - } while (read_size); -exit: - return ret; -} - -/** - * reader_read() - Handles a read call on the reader file descriptor - * - * @filp: The file that the read was performed on - * @buffer: The destination buffer - * @buffer_size: The maximum number of bytes to read - * @offset: The offset into the 'file' to read from. - * - * Note the destination buffer needs to be fully mapped in userspace or the read - * will fault. - * - * Return: - * * The number of bytes read or: - * * -EBADF - the file descriptor did not have an attached reader - * * -EFAULT - memory access fault - * * -EAGAIN - if the file is set to nonblocking reads with O_NONBLOCK and there - * is no data available - * - * Note: The number of bytes read will always be a multiple of the size of an - * entry. - */ -static ssize_t reader_read(struct file *const filp, char __user *const buffer, - size_t const buffer_size, loff_t *const offset) -{ - struct reader *const reader = filp->private_data; - struct reader_changes *changes; - ssize_t ret; - - CSTD_UNUSED(offset); - - if (!reader) - return -EBADF; - - if (buffer_size < sizeof(struct kbase_kinstr_jm_atom_state_change)) - return -ENOBUFS; - -#if KERNEL_VERSION(5, 0, 0) <= LINUX_VERSION_CODE - if (!access_ok(buffer, buffer_size)) - return -EIO; -#else - if (!access_ok(VERIFY_WRITE, buffer, buffer_size)) - return -EIO; -#endif - - changes = &reader->changes; - - mutex_lock(&changes->consumer); - if (!reader_changes_count_locked(changes)) { - if (filp->f_flags & O_NONBLOCK) { - ret = -EAGAIN; - goto exit; - } - - if (wait_event_interruptible(reader->wait_queue, - !!reader_changes_count_locked(changes))) { - ret = -EINTR; - goto exit; - } - } - - ret = reader_changes_copy_to_user(changes, buffer, buffer_size); - -exit: - mutex_unlock(&changes->consumer); - return ret; -} - -/** - * reader_poll() - Handles a poll call on the reader file descriptor - * @file: The file that the poll was performed on - * @wait: The poll table - * - * The results of the poll will be unreliable if there is no mapped memory as - * there is no circular buffer to push atom state changes into. - * - * Return: - * * 0 - no data ready - * * EPOLLIN | EPOLLRDNORM - state changes have been buffered - * * EPOLLHUP | EPOLLERR - IO control arguments were invalid or the file - * descriptor did not have an attached reader. - */ -static __poll_t reader_poll(struct file *const file, struct poll_table_struct *const wait) -{ - struct reader *reader; - struct reader_changes *changes; - __poll_t mask = 0; - - if (unlikely(!file || !wait)) - return EPOLLHUP | EPOLLERR; - - reader = file->private_data; - if (unlikely(!reader)) - return EPOLLHUP | EPOLLERR; - - changes = &reader->changes; - if (reader_changes_count(changes) >= changes->threshold) - return EPOLLIN | EPOLLRDNORM; - - poll_wait(file, &reader->wait_queue, wait); - - if (reader_changes_count(changes) > 0) - mask |= EPOLLIN | EPOLLRDNORM; - - return mask; -} - -/* The file operations virtual function table */ -static const struct file_operations file_operations = { .owner = THIS_MODULE, - .llseek = no_llseek, - .read = reader_read, - .poll = reader_poll, - .release = reader_release }; - -/* The maximum amount of readers that can be created on a context. */ -static const size_t kbase_kinstr_jm_readers_max = 16; - -/** - * kbase_kinstr_jm_release() - Invoked when the reference count is dropped - * @ref: the context reference count - */ -static void kbase_kinstr_jm_release(struct kref *const ref) -{ - struct kbase_kinstr_jm *const ctx = container_of(ref, struct kbase_kinstr_jm, refcount); - - kfree(ctx); -} - -/** - * kbase_kinstr_jm_ref_get() - Reference counts the instrumentation context - * @ctx: the context to reference count - * Return: the reference counted context - */ -static struct kbase_kinstr_jm *kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx) -{ - if (likely(ctx)) - kref_get(&ctx->refcount); - return ctx; -} - -/** - * kbase_kinstr_jm_ref_put() - Dereferences the instrumentation context - * @ctx: the context to lower the reference count on - */ -static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx) -{ - if (likely(ctx)) - kref_put(&ctx->refcount, kbase_kinstr_jm_release); -} - -/** - * kbase_kinstr_jm_readers_add() - Adds a reader to the list of readers - * @ctx: the instrumentation context - * @reader: the reader to add - * - * Return: - * 0 - success - * -ENOMEM - too many readers already added. - */ -static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, - struct reader *const reader) -{ - struct hlist_bl_head *const readers = &ctx->readers; - struct hlist_bl_node *node; - struct reader *temp; - size_t count = 0; - - hlist_bl_lock(readers); - - hlist_bl_for_each_entry_rcu(temp, node, readers, node) - ++count; - - if (kbase_kinstr_jm_readers_max < count) { - hlist_bl_unlock(readers); - return -ENOMEM; - } - - hlist_bl_add_head_rcu(&reader->node, readers); - - hlist_bl_unlock(readers); - - static_branch_inc(&basep_kinstr_jm_reader_static_key); - - return 0; -} - -/** - * kbase_kinstr_jm_readers_del() - Deletes a reader from the list of readers - * @ctx: the instrumentation context - * @reader: the reader to delete - */ -static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, - struct reader *const reader) -{ - struct hlist_bl_head *const readers = &ctx->readers; - - hlist_bl_lock(readers); - hlist_bl_del_rcu(&reader->node); - hlist_bl_unlock(readers); - - static_branch_dec(&basep_kinstr_jm_reader_static_key); -} - -int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, union kbase_kinstr_jm_fd *jm_fd_arg) -{ - struct kbase_kinstr_jm_fd_in const *in; - struct reader *reader; - size_t const change_size = sizeof(struct kbase_kinstr_jm_atom_state_change); - int status; - int fd; - - if (!ctx || !jm_fd_arg) - return -EINVAL; - - in = &jm_fd_arg->in; - - if (!is_power_of_2(in->count)) - return -EINVAL; - - status = reader_init(&reader, ctx, in->count); - if (status < 0) - return status; - - jm_fd_arg->out.version = KBASE_KINSTR_JM_VERSION; - jm_fd_arg->out.size = change_size; - memset(&jm_fd_arg->out.padding, 0, sizeof(jm_fd_arg->out.padding)); - - fd = anon_inode_getfd("[mali_kinstr_jm]", &file_operations, reader, O_CLOEXEC); - if (fd < 0) - reader_term(reader); - - return fd; -} - -int kbase_kinstr_jm_init(struct kbase_kinstr_jm **const out_ctx) -{ - struct kbase_kinstr_jm *ctx = NULL; - - if (!out_ctx) - return -EINVAL; - - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); - if (!ctx) - return -ENOMEM; - - INIT_HLIST_BL_HEAD(&ctx->readers); - kref_init(&ctx->refcount); - - *out_ctx = ctx; - - return 0; -} - -void kbase_kinstr_jm_term(struct kbase_kinstr_jm *const ctx) -{ - kbase_kinstr_jm_ref_put(ctx); -} - -void kbasep_kinstr_jm_atom_state(struct kbase_jd_atom *const katom, - const enum kbase_kinstr_jm_reader_atom_state state) -{ - struct kbase_context *const kctx = katom->kctx; - struct kbase_kinstr_jm *const ctx = kctx->kinstr_jm; - const u8 id = kbase_jd_atom_id(kctx, katom); - struct kbase_kinstr_jm_atom_state_change change = { .timestamp = ktime_get_raw_ns(), - .atom = id, - .state = state }; - struct reader *reader; - struct hlist_bl_node *node; - - WARN(KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT < state || 0 > state, - PR_ "unsupported katom (%u) state (%i)", id, state); - - switch (state) { - case KBASE_KINSTR_JM_READER_ATOM_STATE_START: - change.data.start.slot = katom->slot_nr; - break; - default: - break; - } - - rcu_read_lock(); - hlist_bl_for_each_entry_rcu(reader, node, &ctx->readers, node) - reader_changes_push(&reader->changes, &change, &reader->wait_queue); - rcu_read_unlock(); -} - -KBASE_EXPORT_TEST_API(kbasep_kinstr_jm_atom_state); - -void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const katom) -{ - struct kbase_context *const kctx = katom->kctx; - struct kbase_device *const kbdev = kctx->kbdev; - const unsigned int slot = katom->slot_nr; - struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); - - BUILD_BUG_ON(SLOT_RB_SIZE != 2); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (WARN_ON(slot >= GPU_MAX_JOB_SLOTS)) - return; - if (WARN_ON(!submitted)) - return; - - if (submitted == katom) - kbase_kinstr_jm_atom_state_start(katom); -} - -void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const katom) -{ - struct kbase_context *const kctx = katom->kctx; - struct kbase_device *const kbdev = kctx->kbdev; - const unsigned int slot = katom->slot_nr; - struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); - struct kbase_jd_atom *const queued = kbase_gpu_inspect(kbdev, slot, 1); - - BUILD_BUG_ON(SLOT_RB_SIZE != 2); - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (WARN_ON(slot >= GPU_MAX_JOB_SLOTS)) - return; - if (WARN_ON(!submitted)) - return; - if (WARN_ON((submitted != katom) && (queued != katom))) - return; - - if (queued == katom) - return; - - if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) - kbase_kinstr_jm_atom_state_stop(katom); - if (queued && queued->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) - kbase_kinstr_jm_atom_state_start(queued); -} diff --git a/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.h b/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.h deleted file mode 100644 index ae5b63f0372f..000000000000 --- a/drivers/gpu/arm/valhall/mali_kbase_kinstr_jm.h +++ /dev/null @@ -1,257 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * mali_kbase_kinstr_jm.h - * Kernel driver public interface to job manager atom tracing. This API provides - * a method to get the atom state changes into user space. - * - * The flow of operation is: - * - * | kernel | user | - * | ----------------------------------- | ----------------------------------- | - * | Initialize API with | | - * | kbase_kinstr_jm_init() | | - * | | | - * | Kernel code injects states with | | - * | kbase_kinstr_jm_atom_state_*() APIs | | - * | | Call ioctl() to get file descriptor | - * | | via KBASE_IOCTL_KINSTR_JM_FD | - * | Allocates a reader attached to FD | | - * | Allocates circular buffer and | | - * | patches, via ASM goto, the | | - * | kbase_kinstr_jm_atom_state_*() | | - * | | loop: | - * | | Call poll() on FD for POLLIN | - * | When threshold of changes is hit, | | - * | the poll is interrupted with | | - * | POLLIN. If circular buffer is | | - * | full then store the missed count | | - * | and interrupt poll | Call read() to get data from | - * | | circular buffer via the fd | - * | Kernel advances tail of circular | | - * | buffer | | - * | | Close file descriptor | - * | Deallocates circular buffer | | - * | | | - * | Terminate API with | | - * | kbase_kinstr_jm_term() | | - * - * All tracepoints are guarded on a static key. The static key is activated when - * a user space reader gets created. This means that there is negligible cost - * inserting the tracepoints into code when there are no readers. - */ - -#ifndef _KBASE_KINSTR_JM_H_ -#define _KBASE_KINSTR_JM_H_ - -#include - -#ifdef __KERNEL__ -#include -#include -#else -/* empty wrapper macros for userspace */ -#define static_branch_unlikely(key) (1) -#endif /* __KERNEL__ */ - -/* Forward declarations */ -struct kbase_context; -struct kbase_kinstr_jm; -struct kbase_jd_atom; -union kbase_kinstr_jm_fd; - -/** - * kbase_kinstr_jm_init() - Initialise an instrumentation job manager context. - * @ctx: Non-NULL pointer to where the pointer to the created context will - * be stored on success. - * - * Return: 0 on success, else error code. - */ -int kbase_kinstr_jm_init(struct kbase_kinstr_jm **ctx); - -/** - * kbase_kinstr_jm_term() - Terminate an instrumentation job manager context. - * @ctx: Pointer to context to be terminated. - */ -void kbase_kinstr_jm_term(struct kbase_kinstr_jm *ctx); - -/** - * kbase_kinstr_jm_get_fd() - Retrieves a file descriptor that can be used to - * read the atom state changes from userspace - * - * @ctx: Pointer to the initialized context - * @jm_fd_arg: Pointer to the union containing the in/out params - * Return: -1 on failure, valid file descriptor on success - */ -int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, union kbase_kinstr_jm_fd *jm_fd_arg); - -/** - * kbasep_kinstr_jm_atom_state() - Signifies that an atom has changed state - * @atom: The atom that has changed state - * @state: The new state of the atom - * - * This performs the actual storage of the state ready for user space to - * read the data. It is only called when the static key is enabled from - * kbase_kinstr_jm_atom_state(). There is almost never a need to invoke this - * function directly. - */ -void kbasep_kinstr_jm_atom_state(struct kbase_jd_atom *const atom, - const enum kbase_kinstr_jm_reader_atom_state state); - -/* Allows ASM goto patching to reduce tracing overhead. This is - * incremented/decremented when readers are created and terminated. This really - * shouldn't be changed externally, but if you do, make sure you use - * a static_key_inc()/static_key_dec() pair. - */ -extern struct static_key_false basep_kinstr_jm_reader_static_key; - -/** - * kbase_kinstr_jm_atom_state() - Signifies that an atom has changed state - * @atom: The atom that has changed state - * @state: The new state of the atom - * - * This uses a static key to reduce overhead when tracing is disabled - */ -static inline void kbase_kinstr_jm_atom_state(struct kbase_jd_atom *const atom, - const enum kbase_kinstr_jm_reader_atom_state state) -{ - if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) - kbasep_kinstr_jm_atom_state(atom, state); -} - -/** - * kbase_kinstr_jm_atom_state_queue() - Signifies that an atom has entered a - * hardware or software queue. - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_state_queue(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state(atom, KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE); -} - -/** - * kbase_kinstr_jm_atom_state_start() - Signifies that work has started on an - * atom - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_state_start(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state(atom, KBASE_KINSTR_JM_READER_ATOM_STATE_START); -} - -/** - * kbase_kinstr_jm_atom_state_stop() - Signifies that work has stopped on an - * atom - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_state_stop(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state(atom, KBASE_KINSTR_JM_READER_ATOM_STATE_STOP); -} - -/** - * kbase_kinstr_jm_atom_state_complete() - Signifies that all work has completed - * on an atom - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_state_complete(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state(atom, KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE); -} - -/** - * kbase_kinstr_jm_atom_queue() - A software *or* hardware atom is queued for - * execution - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_queue(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state_queue(atom); -} - -/** - * kbase_kinstr_jm_atom_complete() - A software *or* hardware atom is fully - * completed - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_complete(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state_complete(atom); -} - -/** - * kbase_kinstr_jm_atom_sw_start() - A software atom has started work - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_sw_start(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state_start(atom); -} - -/** - * kbase_kinstr_jm_atom_sw_stop() - A software atom has stopped work - * @atom: The atom that has changed state - */ -static inline void kbase_kinstr_jm_atom_sw_stop(struct kbase_jd_atom *const atom) -{ - kbase_kinstr_jm_atom_state_stop(atom); -} - -/** - * kbasep_kinstr_jm_atom_hw_submit() - A hardware atom has been submitted - * @atom: The atom that has been submitted - * - * This private implementation should not be called directly, it is protected - * by a static key in kbase_kinstr_jm_atom_hw_submit(). Use that instead. - */ -void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const atom); - -/** - * kbase_kinstr_jm_atom_hw_submit() - A hardware atom has been submitted - * @atom: The atom that has been submitted - */ -static inline void kbase_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const atom) -{ - if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) - kbasep_kinstr_jm_atom_hw_submit(atom); -} - -/** - * kbasep_kinstr_jm_atom_hw_release() - A hardware atom has been released - * @atom: The atom that has been released - * - * This private implementation should not be called directly, it is protected - * by a static key in kbase_kinstr_jm_atom_hw_release(). Use that instead. - */ -void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const atom); - -/** - * kbase_kinstr_jm_atom_hw_release() - A hardware atom has been released - * @atom: The atom that has been released - */ -static inline void kbase_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const atom) -{ - if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) - kbasep_kinstr_jm_atom_hw_release(atom); -} - -#endif /* _KBASE_KINSTR_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem.c b/drivers/gpu/arm/valhall/mali_kbase_mem.c index cb2e230fbc91..4f5e1980d772 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem.c @@ -246,9 +246,6 @@ int kbase_mem_init(struct kbase_device *kbdev) } } - kbase_mem_pool_group_config_set_max_size(&kbdev->mem_pool_defaults, - KBASE_MEM_POOL_MAX_SIZE_KCTX); - spin_lock_init(&kbdev->gpu_mem_usage_lock); kbdev->process_root = RB_ROOT; kbdev->dma_buf_root = RB_ROOT; @@ -282,20 +279,37 @@ int kbase_mem_init(struct kbase_device *kbdev) } } of_node_put(mgm_node); + if (unlikely(err)) + goto init_fail; } #endif - if (likely(!err)) { - struct kbase_mem_pool_group_config mem_pool_defaults; + err = kbase_mem_pool_init_no_reclaim(&kbdev->pgd_mem_pool, BASE_PGD_MEM_POOL_MAX_SIZE_KBDEV, + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, 0, kbdev); - kbase_mem_pool_group_config_set_max_size(&mem_pool_defaults, - KBASE_MEM_POOL_MAX_SIZE_KBDEV); + if (unlikely(err)) + goto init_fail; - err = kbase_mem_pool_group_init(&kbdev->mem_pools, kbdev, &mem_pool_defaults, NULL); - if (likely(!err)) - return err; - } + err = kbase_mem_pool_init_no_reclaim(&kbdev->fw_mem_pools.small, + KBASE_FW_MEM_POOL_MAX_SIZE_KBDEV, + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, 0, kbdev); + if (unlikely(err)) + goto term_kbase_pgd_mem_pool; + + err = kbase_mem_pool_init_no_reclaim(&kbdev->fw_mem_pools.large, + KBASE_FW_MEM_POOL_MAX_SIZE_KBDEV >> + (KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER - + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER), + KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, 0, kbdev); + + if (likely(!err)) + return err; + + kbase_mem_pool_term(&kbdev->fw_mem_pools.small); +term_kbase_pgd_mem_pool: + kbase_mem_pool_term(&kbdev->pgd_mem_pool); +init_fail: kmem_cache_destroy(kbdev->page_metadata_slab); kbdev->page_metadata_slab = NULL; page_metadata_slab_fail: @@ -324,7 +338,9 @@ void kbase_mem_term(struct kbase_device *kbdev) if (pages != 0) dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages); - kbase_mem_pool_group_term(&kbdev->mem_pools); + kbase_mem_pool_term(&kbdev->pgd_mem_pool); + kbase_mem_pool_term(&kbdev->fw_mem_pools.small); + kbase_mem_pool_term(&kbdev->fw_mem_pools.large); kbase_mem_migrate_term(kbdev); @@ -848,11 +864,10 @@ static int kbase_do_syncset(struct kbase_context *kctx, struct basep_syncset *ss kbase_gpu_vm_lock(kctx); /* find the region where the virtual address is contained */ - reg = kbase_region_tracker_find_region_enclosing_address(kctx, - sset->mem_handle.basep.handle); + reg = kbase_region_tracker_find_region_enclosing_address(kctx, sset->gpu_va); if (kbase_is_region_invalid_or_free(reg)) { dev_warn(kctx->kbdev->dev, "Can't find a valid region at VA 0x%016llX", - sset->mem_handle.basep.handle); + sset->gpu_va); err = -EINVAL; goto out_unlock; } @@ -938,8 +953,8 @@ int kbase_sync_now(struct kbase_context *kctx, struct basep_syncset *sset) KBASE_DEBUG_ASSERT(kctx != NULL); KBASE_DEBUG_ASSERT(sset != NULL); - if (sset->mem_handle.basep.handle & ~PAGE_MASK) { - dev_warn(kctx->kbdev->dev, "mem_handle: passed parameter is invalid"); + if (sset->gpu_va & ~PAGE_MASK) { + dev_warn(kctx->kbdev->dev, "gpu_va 0x%016llX parameter is invalid", sset->gpu_va); return -EINVAL; } @@ -1003,7 +1018,6 @@ int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *re goto out; } -#if MALI_USE_CSF if (((kbase_bits_to_zone(reg->flags)) == FIXED_VA_ZONE) || ((kbase_bits_to_zone(reg->flags)) == EXEC_FIXED_VA_ZONE)) { if (reg->flags & KBASE_REG_FIXED_ADDRESS) @@ -1011,7 +1025,6 @@ int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *re else atomic64_dec(&kctx->num_fixable_allocs); } -#endif /* This will also free the physical pages */ kbase_free_alloced_region(reg); @@ -1137,12 +1150,6 @@ int kbase_update_region_flags(struct kbase_context *kctx, struct kbase_va_region reg->flags |= KBASE_REG_SHARE_IN; } -#if !MALI_USE_CSF - if (flags & BASE_MEM_TILER_ALIGN_TOP) - reg->flags |= KBASE_REG_TILER_ALIGN_TOP; -#endif /* !MALI_USE_CSF */ - -#if MALI_USE_CSF if (flags & BASE_MEM_CSF_EVENT) { reg->flags |= KBASE_REG_CSF_EVENT; reg->flags |= KBASE_REG_PERMANENT_KERNEL_MAPPING; @@ -1155,7 +1162,6 @@ int kbase_update_region_flags(struct kbase_context *kctx, struct kbase_va_region reg->flags &= ~KBASE_REG_GPU_CACHED; } } -#endif /* Set up default MEMATTR usage */ if (!(reg->flags & KBASE_REG_GPU_CACHED)) { @@ -1168,12 +1174,10 @@ int kbase_update_region_flags(struct kbase_context *kctx, struct kbase_va_region "Can't allocate GPU uncached memory due to MMU in Legacy Mode\n"); return -EINVAL; } -#if MALI_USE_CSF } else if (reg->flags & KBASE_REG_CSF_EVENT) { WARN_ON(!(reg->flags & KBASE_REG_SHARE_BOTH)); reg->flags |= KBASE_REG_MEMATTR_INDEX(KBASE_MEMATTR_INDEX_SHARED); -#endif } else if (kctx->kbdev->system_coherency == COHERENCY_ACE && (reg->flags & KBASE_REG_SHARE_BOTH)) { reg->flags |= KBASE_REG_MEMATTR_INDEX(KBASE_MEMATTR_INDEX_DEFAULT_ACE); @@ -1193,10 +1197,8 @@ int kbase_update_region_flags(struct kbase_context *kctx, struct kbase_va_region if (flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) reg->flags |= KBASE_REG_GPU_VA_SAME_4GB_PAGE; -#if MALI_USE_CSF if (flags & BASE_MEM_FIXED) reg->flags |= KBASE_REG_FIXED_ADDRESS; -#endif if (flags & BASEP_MEM_ACTIVE_JIT_ALLOC) reg->flags |= BASEP_MEM_ACTIVE_JIT_ALLOC; @@ -2085,26 +2087,12 @@ bool kbase_check_alloc_flags(struct kbase_context *kctx, unsigned long flags) (flags & (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) return false; -#if !MALI_USE_CSF - /* GPU executable memory also cannot have the top of its initial - * commit aligned to 'extension' - */ - if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & BASE_MEM_TILER_ALIGN_TOP)) - return false; -#endif /* !MALI_USE_CSF */ - /* To have an allocation lie within a 4GB chunk is required only for * TLS memory, which will never be used to contain executable code. */ if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & BASE_MEM_PROT_GPU_EX)) return false; -#if !MALI_USE_CSF - /* TLS memory should also not be used for tiler heap */ - if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & BASE_MEM_TILER_ALIGN_TOP)) - return false; -#endif /* !MALI_USE_CSF */ - /* GPU should have at least read or write access otherwise there is no * reason for allocating. */ @@ -2127,13 +2115,11 @@ bool kbase_check_alloc_flags(struct kbase_context *kctx, unsigned long flags) (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) return false; -#if MALI_USE_CSF if ((flags & BASE_MEM_SAME_VA) && (flags & (BASE_MEM_FIXABLE | BASE_MEM_FIXED))) return false; if ((flags & BASE_MEM_FIXABLE) && (flags & BASE_MEM_FIXED)) return false; -#endif /* Cannot be set only allocation, only with base_mem_set */ if ((flags & BASE_MEM_DONT_NEED) && @@ -2155,30 +2141,13 @@ bool kbase_check_alloc_flags(struct kbase_context *kctx, unsigned long flags) (mali_kbase_supports_reject_alloc_mem_unused_bit_7(kctx->api_version))) return false; -#if !MALI_USE_CSF - if ((flags & BASE_MEM_UNUSED_BIT_8) && - (mali_kbase_supports_reject_alloc_mem_unused_bit_8(kctx->api_version))) - return false; - - if ((flags & BASE_MEM_UNUSED_BIT_19) && - (mali_kbase_supports_reject_alloc_mem_unused_bit_19(kctx->api_version))) - return false; - -#else if ((flags & BASE_MEM_UNUSED_BIT_20) && (mali_kbase_supports_reject_alloc_mem_unused_bit_20(kctx->api_version))) return false; -#endif /* !MALI_USE_CSF */ if ((flags & BASE_MEM_UNUSED_BIT_27) && (mali_kbase_supports_reject_alloc_mem_unused_bit_27(kctx->api_version))) return false; -#if !MALI_USE_CSF - - if ((flags & BASE_MEM_UNUSED_BIT_29) && - (mali_kbase_supports_reject_alloc_mem_unused_bit_29(kctx->api_version))) - return false; -#endif /* !MALI_USE_CSF */ return true; } @@ -2201,15 +2170,9 @@ bool kbase_check_import_flags(unsigned long flags) if (flags & BASE_MEM_GROW_ON_GPF) return false; -#if MALI_USE_CSF /* Imported memory cannot be fixed */ if ((flags & (BASE_MEM_FIXED | BASE_MEM_FIXABLE))) return false; -#else - /* Imported memory cannot be aligned to the end of its initial commit */ - if (flags & BASE_MEM_TILER_ALIGN_TOP) - return false; -#endif /* !MALI_USE_CSF */ /* GPU should have at least read or write access otherwise there is no * reason for importing. @@ -2267,59 +2230,12 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, u64 return -EINVAL; } -#if !MALI_USE_CSF - if ((flags & BASE_MEM_TILER_ALIGN_TOP) && (test_reg.extension == 0)) { - dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP but extension == 0\n"); - return -EINVAL; - } - - if (!(flags & (BASE_MEM_GROW_ON_GPF | BASE_MEM_TILER_ALIGN_TOP)) && - test_reg.extension != 0) { - dev_warn( - dev, KBASE_MSG_PRE - "neither BASE_MEM_GROW_ON_GPF nor BASE_MEM_TILER_ALIGN_TOP set but extension != 0\n"); - return -EINVAL; - } -#else if (!(flags & BASE_MEM_GROW_ON_GPF) && test_reg.extension != 0) { dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GROW_ON_GPF not set but extension != 0\n"); return -EINVAL; } -#endif /* !MALI_USE_CSF */ -#if !MALI_USE_CSF - /* BASE_MEM_TILER_ALIGN_TOP memory has a number of restrictions */ - if (flags & BASE_MEM_TILER_ALIGN_TOP) { -#define KBASE_MSG_PRE_FLAG KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP and " - unsigned long small_extension; - - if (large_extension > BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES) { - dev_warn(dev, KBASE_MSG_PRE_FLAG "extension==%lld pages exceeds limit %lld", - (unsigned long long)large_extension, - BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES); - return -EINVAL; - } - /* For use with is_power_of_2, which takes unsigned long, so - * must ensure e.g. on 32-bit kernel it'll fit in that type - */ - small_extension = (unsigned long)large_extension; - - if (!is_power_of_2(small_extension)) { - dev_warn(dev, KBASE_MSG_PRE_FLAG "extension==%ld not a non-zero power of 2", - small_extension); - return -EINVAL; - } - - if (commit_pages > large_extension) { - dev_warn(dev, KBASE_MSG_PRE_FLAG "commit_pages==%ld exceeds extension==%ld", - (unsigned long)commit_pages, (unsigned long)large_extension); - return -EINVAL; - } -#undef KBASE_MSG_PRE_FLAG - } -#else CSTD_UNUSED(commit_pages); -#endif /* !MALI_USE_CSF */ if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (va_pages > (BASE_MEM_PFN_MASK_4GB + 1))) { dev_warn( @@ -2343,9 +2259,7 @@ KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); void kbase_gpu_vm_lock_with_pmode_sync(struct kbase_context *kctx) { -#if MALI_USE_CSF down_read(&kctx->kbdev->csf.mmu_sync_sem); -#endif kbase_gpu_vm_lock(kctx); } @@ -2359,9 +2273,7 @@ KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); void kbase_gpu_vm_unlock_with_pmode_sync(struct kbase_context *kctx) { kbase_gpu_vm_unlock(kctx); -#if MALI_USE_CSF up_read(&kctx->kbdev->csf.mmu_sync_sem); -#endif } #if IS_ENABLED(CONFIG_DEBUG_FS) @@ -2524,17 +2436,11 @@ static int kbase_jit_debugfs_used_get(struct kbase_jit_debugfs_data *data) struct kbase_context *kctx = data->kctx; struct kbase_va_region *reg; -#if !MALI_USE_CSF - mutex_lock(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ mutex_lock(&kctx->jit_evict_lock); list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { data->active_value += reg->used_pages; } mutex_unlock(&kctx->jit_evict_lock); -#if !MALI_USE_CSF - mutex_unlock(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ return 0; } @@ -2550,9 +2456,6 @@ static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data) struct kbase_context *kctx = data->kctx; struct kbase_va_region *reg; -#if !MALI_USE_CSF - mutex_lock(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ kbase_gpu_vm_lock(kctx); mutex_lock(&kctx->jit_evict_lock); list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { @@ -2570,9 +2473,6 @@ static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data) } mutex_unlock(&kctx->jit_evict_lock); kbase_gpu_vm_unlock(kctx); -#if !MALI_USE_CSF - mutex_unlock(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ return 0; } @@ -2677,14 +2577,9 @@ int kbase_jit_init(struct kbase_context *kctx) INIT_LIST_HEAD(&kctx->jit_destroy_head); INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); -#if MALI_USE_CSF mutex_init(&kctx->csf.kcpu_queues.jit_lock); INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_cmds_head); INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_blocked_queues); -#else /* !MALI_USE_CSF */ - INIT_LIST_HEAD(&kctx->jctx.jit_atoms_head); - INIT_LIST_HEAD(&kctx->jctx.jit_pending_alloc); -#endif /* MALI_USE_CSF */ mutex_unlock(&kctx->jit_evict_lock); return 0; @@ -2702,16 +2597,6 @@ static bool meet_size_and_tiler_align_top_requirements(const struct kbase_va_reg if (walker->nr_pages != info->va_pages) meet_reqs = false; -#if !MALI_USE_CSF - if (meet_reqs && (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP)) { - size_t align = info->extension; - size_t align_mask = align - 1; - - if ((walker->start_pfn + info->commit_pages) & align_mask) - meet_reqs = false; - } -#endif /* !MALI_USE_CSF */ - return meet_reqs; } @@ -2729,9 +2614,6 @@ static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, size_t to_free = 0u; size_t max_allowed_pages = old_pages; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kctx->reg_lock); /* Is this a JIT allocation that has been reported on? */ @@ -2846,9 +2728,6 @@ static size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, size_t pages_ struct kbase_va_region *reg, *tmp; size_t total_freed = 0; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kctx->reg_lock); lockdep_assert_held(&kctx->jit_evict_lock); @@ -3048,9 +2927,6 @@ void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, size_t needed_pa size_t jit_backing = 0; size_t pages_to_trim = 0; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kctx->reg_lock); lockdep_assert_held(&kctx->jit_evict_lock); @@ -3096,11 +2972,7 @@ void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, size_t needed_pa static bool jit_allow_allocate(struct kbase_context *kctx, const struct base_jit_alloc_info *info, bool ignore_pressure_limit) { -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#else /* MALI_USE_CSF */ lockdep_assert_held(&kctx->csf.kcpu_queues.jit_lock); -#endif /* !MALI_USE_CSF */ #if MALI_JIT_PRESSURE_LIMIT_BASE if (!ignore_pressure_limit && @@ -3187,11 +3059,7 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, */ const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_SYNC; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#else /* MALI_USE_CSF */ lockdep_assert_held(&kctx->csf.kcpu_queues.jit_lock); -#endif /* !MALI_USE_CSF */ if (!jit_allow_allocate(kctx, info, ignore_pressure_limit)) return NULL; @@ -3330,11 +3198,6 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, BASEP_MEM_ACTIVE_JIT_ALLOC; u64 gpu_addr; -#if !MALI_USE_CSF - if (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP) - flags |= BASE_MEM_TILER_ALIGN_TOP; -#endif /* !MALI_USE_CSF */ - flags |= kbase_mem_group_id_set(kctx->jit_group_id); #if MALI_JIT_PRESSURE_LIMIT_BASE if (!ignore_pressure_limit) { @@ -3431,11 +3294,7 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) { u64 old_pages; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#else /* MALI_USE_CSF */ lockdep_assert_held(&kctx->csf.kcpu_queues.jit_lock); -#endif /* !MALI_USE_CSF */ /* JIT id not immediately available here, so use 0u */ trace_mali_jit_free(reg, 0u); @@ -3669,10 +3528,6 @@ void kbase_jit_report_update_pressure(struct kbase_context *kctx, struct kbase_v { u64 diff; -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ - trace_mali_jit_report_pressure( reg, new_used_pages, kctx->jit_current_phys_pressure + new_used_pages - reg->used_pages, flags); @@ -4533,9 +4388,6 @@ void kbase_user_buf_from_dma_mapped_to_pinned(struct kbase_context *kctx, dev_dbg(kctx->kbdev->dev, "%s %pK in kctx %pK\n", __func__, (void *)reg, (void *)kctx); if (WARN_ON(reg->gpu_alloc->imported.user_buf.state != KBASE_USER_BUF_STATE_DMA_MAPPED)) return; -#if !MALI_USE_CSF - kbase_mem_shrink_cpu_mapping(kctx, reg, 0, reg->gpu_alloc->nents); -#endif kbase_user_buf_dma_unmap_pages(kctx, reg); reg->gpu_alloc->imported.user_buf.state = KBASE_USER_BUF_STATE_PINNED; diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem.h b/drivers/gpu/arm/valhall/mali_kbase_mem.h index 455c3efc3bd2..3a92ce9ec8b6 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem.h +++ b/drivers/gpu/arm/valhall/mali_kbase_mem.h @@ -174,7 +174,6 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int /* Normal memory, inner non-cacheable, outer non-cacheable (ARMv8 mode only) */ #define KBASE_MEMATTR_INDEX_NON_CACHEABLE 5 -#if MALI_USE_CSF /* Set to shared memory, that is inner cacheable on ACE and inner or outer * shared, otherwise inner non-cacheable. * Outer cacheable if inner or outer shared, otherwise outer non-cacheable. @@ -187,14 +186,12 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int /* Normal memory, shared between MCU and Host */ #define KBASE_MEMATTR_INDEX_SHARED 6 -#endif #define KBASE_REG_PROTECTED (1ul << 19) /* Imported buffer is padded? */ #define KBASE_REG_IMPORT_PAD (1ul << 21) -#if MALI_USE_CSF /* CSF event memory */ #define KBASE_REG_CSF_EVENT (1ul << 22) /* Bit 23 is reserved. @@ -202,17 +199,6 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int * Do not remove, use the next unreserved bit for new flags */ #define KBASE_REG_RESERVED_BIT_23 (1ul << 23) -#else -/* Bit 22 is reserved. - * - * Do not remove, use the next unreserved bit for new flags - */ -#define KBASE_REG_RESERVED_BIT_22 (1ul << 22) -/* The top of the initial commit is aligned to extension pages. - * Extent must be a power of 2 - */ -#define KBASE_REG_TILER_ALIGN_TOP (1ul << 23) -#endif /* MALI_USE_CSF */ /* Bit 24 is currently unused and is available for use for a new flag */ @@ -241,15 +227,11 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int */ #define KBASE_REG_HEAP_INFO_IS_SIZE (1ul << 27) -#if MALI_USE_CSF /* This flag only applies to allocations in the EXEC_FIXED_VA and FIXED_VA * memory zones, and it determines whether they were created with a fixed * GPU VA address requested by the user. */ #define KBASE_REG_FIXED_ADDRESS (1ul << 29) -#else -#define KBASE_REG_RESERVED_BIT_29 (1ul << 29) -#endif /* * A CPU mapping */ @@ -975,11 +957,25 @@ static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, struct k */ #define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT) +/* + * Max size for kbdev pgd memory pool (in pages) + */ +#define BASE_PGD_MEM_POOL_MAX_SIZE_KBDEV (SZ_2M >> PAGE_SHIFT) + +/* + * Max size for kbdev fw memory pool (in pages) + */ +#define KBASE_FW_MEM_POOL_MAX_SIZE_KBDEV ((4 * SZ_2M) >> PAGE_SHIFT) /* * Max size for kctx memory pool (in pages) */ #define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) +/* + * Max size of kctx pgd memory pool (in pages) + */ +#define KBASE_PGD_MM_POOLMAX_SIZE_KCTX (SZ_2M >> PAGE_SHIFT) + /* * The order required for a 2MB page allocation (2^order * PAGE_SIZE = 2MB) */ @@ -1022,13 +1018,12 @@ kbase_mem_pool_config_get_max_size(const struct kbase_mem_pool_config *const con /** * kbase_mem_pool_init - Create a memory pool for a kbase device * @pool: Memory pool to initialize - * @config: Initial configuration for the memory pool + * @max_size: Maximum size for the memory pool * @order: Page order for physical page size (order=0 => small page, order != 0 => 2MB) * @group_id: A memory group ID to be passed to a platform-specific * memory group manager, if present. * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). * @kbdev: Kbase device where memory is used - * @next_pool: Pointer to the next pool or NULL. * * Allocations from @pool are in whole pages. Each @pool has a free list where * pages can be quickly allocated from. The free list is initially empty and @@ -1046,9 +1041,26 @@ kbase_mem_pool_config_get_max_size(const struct kbase_mem_pool_config *const con * * Return: 0 on success, negative -errno on error */ -int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool_config *config, - unsigned int order, int group_id, struct kbase_device *kbdev, - struct kbase_mem_pool *next_pool); +int kbase_mem_pool_init(struct kbase_mem_pool *pool, size_t max_size, unsigned int order, + int group_id, struct kbase_device *kbdev); + +/** + * kbase_mem_pool_init_no_reclaim - Create a memory pool for a kbase device, with no reclaim. + * @pool: Memory pool to initialize + * @max_size: Maximum size for the memory pool + * @order: Page order for physical page size (order=0 => small page, order != 0 => 2MB) + * @group_id: A memory group ID to be passed to a platform-specific + * memory group manager, if present. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @kbdev: Kbase device where memory is used + * + * Identical to kbase_mem_pool_init, except the pool does not support reclaiming of pages. + * For that reason it also omits support of the next_pool feature. + * + * Return: 0 on success, negative -errno on error + */ +int kbase_mem_pool_init_no_reclaim(struct kbase_mem_pool *pool, size_t max_size, unsigned int order, + int group_id, struct kbase_device *kbdev); /** * kbase_mem_pool_term - Destroy a memory pool @@ -2000,9 +2012,6 @@ void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, size_t needed_pa static inline void kbase_jit_request_phys_increase_locked(struct kbase_context *kctx, size_t needed_pages) { -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kctx->reg_lock); lockdep_assert_held(&kctx->jit_evict_lock); @@ -2038,9 +2047,6 @@ static inline void kbase_jit_request_phys_increase_locked(struct kbase_context * */ static inline void kbase_jit_request_phys_increase(struct kbase_context *kctx, size_t needed_pages) { -#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -#endif /* !MALI_USE_CSF */ lockdep_assert_held(&kctx->reg_lock); mutex_lock(&kctx->jit_evict_lock); @@ -2395,7 +2401,6 @@ static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) */ void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc); -#if MALI_USE_CSF /** * kbase_link_event_mem_page - Add the new event memory region to the per * context list of event pages. @@ -2451,7 +2456,6 @@ int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev); * @kbdev: Pointer to the kbase device */ void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev); -#endif /** * kbase_mem_umm_map - Map dma-buf diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_flags.h b/drivers/gpu/arm/valhall/mali_kbase_mem_flags.h index 84948f9fa445..3ffd7701bd0b 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_flags.h +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_flags.h @@ -22,11 +22,7 @@ #ifndef _KBASE_MEM_FLAGS_H_ #define _KBASE_MEM_FLAGS_H_ -#if MALI_USE_CSF #include -#else -#include -#endif /* Will be permanently mapped in kernel space. * Flag is only allowed on allocations originating from kbase. diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c b/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c index 321df3afd279..d7301653788e 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c @@ -97,10 +97,8 @@ MODULE_PARM_DESC( "Note that this should only be enabled for testing worst case memory usage " "and should not be enabled in production"); -#if MALI_USE_CSF static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, struct vm_area_struct *vma); static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, struct vm_area_struct *vma); -#endif static int kbase_vmap_phy_pages(struct kbase_context *kctx, struct kbase_va_region *reg, u64 offset_bytes, size_t size, struct kbase_vmap_struct *map, @@ -132,7 +130,6 @@ static bool is_process_exiting(struct vm_area_struct *vma) */ static struct kbase_va_region *kbase_find_event_mem_region(struct kbase_context *kctx, u64 gpu_addr) { -#if MALI_USE_CSF u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; struct kbase_va_region *reg; @@ -149,10 +146,6 @@ static struct kbase_va_region *kbase_find_event_mem_region(struct kbase_context return reg; } } -#else - CSTD_UNUSED(kctx); - CSTD_UNUSED(gpu_addr); -#endif return NULL; } @@ -310,13 +303,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages dev_dbg(dev, "Allocating %lld va_pages, %lld commit_pages, %lld extension, 0x%llX flags\n", va_pages, commit_pages, extension, *flags); -#if MALI_USE_CSF if (!(*flags & BASE_MEM_FIXED)) *gpu_va = 0; /* return 0 on failure */ -#else - if (!(*flags & BASEP_MEM_FLAG_MAP_FIXED)) - *gpu_va = 0; /* return 0 on failure */ -#endif else dev_dbg(dev, "Keeping requested GPU VA of 0x%llx\n", (unsigned long long)*gpu_va); @@ -370,7 +358,6 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages /* find out which VA zone to use */ if (*flags & BASE_MEM_SAME_VA) zone = SAME_VA_ZONE; -#if MALI_USE_CSF /* fixed va_zone always exists */ else if (*flags & (BASE_MEM_FIXED | BASE_MEM_FIXABLE)) { if (*flags & BASE_MEM_PROT_GPU_EX) { @@ -378,9 +365,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages } else { zone = FIXED_VA_ZONE; } - } -#endif - else if ((*flags & BASE_MEM_PROT_GPU_EX) && kbase_has_exec_va_zone(kctx)) { + } else if ((*flags & BASE_MEM_PROT_GPU_EX) && kbase_has_exec_va_zone(kctx)) { zone = EXEC_VA_ZONE; } else { zone = CUSTOM_VA_ZONE; @@ -415,10 +400,6 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages * reg->extension. */ reg->extension = extension; -#if !MALI_USE_CSF - } else if (*flags & BASE_MEM_TILER_ALIGN_TOP) { - reg->extension = extension; -#endif /* !MALI_USE_CSF */ } else { reg->extension = 0; } @@ -480,10 +461,6 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages } if (*gpu_va) align = 1; -#if !MALI_USE_CSF - if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) - align = 1; -#endif /* !MALI_USE_CSF */ } if (kbase_gpu_mmap(kctx, reg, *gpu_va, va_pages, align, mmu_sync_info) != 0) { dev_warn(dev, "Failed to map memory on GPU"); @@ -507,12 +484,10 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages kbase_gpu_vm_unlock_with_pmode_sync(kctx); -#if MALI_USE_CSF if (*flags & BASE_MEM_FIXABLE) atomic64_inc(&kctx->num_fixable_allocs); else if (*flags & BASE_MEM_FIXED) atomic64_inc(&kctx->num_fixed_allocs); -#endif return reg; @@ -617,13 +592,8 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co if (KBASE_REG_PROTECTED & reg->flags) *out |= BASE_MEM_PROTECTED; } -#if !MALI_USE_CSF - if (KBASE_REG_TILER_ALIGN_TOP & reg->flags) - *out |= BASE_MEM_TILER_ALIGN_TOP; -#endif /* !MALI_USE_CSF */ if (!(KBASE_REG_GPU_CACHED & reg->flags)) *out |= BASE_MEM_UNCACHED_GPU; -#if MALI_USE_CSF if (KBASE_REG_CSF_EVENT & reg->flags) *out |= BASE_MEM_CSF_EVENT; if ((kbase_bits_to_zone(reg->flags) == FIXED_VA_ZONE) || @@ -633,7 +603,6 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co else *out |= BASE_MEM_FIXABLE; } -#endif /* MALI_USE_CSF */ if (KBASE_REG_GPU_VA_SAME_4GB_PAGE & reg->flags) *out |= BASE_MEM_GPU_VA_SAME_4GB_PAGE; if (mali_kbase_supports_query_mem_import_sync_on_map_unmap(kctx->api_version)) { @@ -741,13 +710,11 @@ static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s kctx = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_context, reclaim); -#if MALI_USE_CSF if (!down_read_trylock(&kctx->kbdev->csf.mmu_sync_sem)) { dev_warn(kctx->kbdev->dev, "Can't shrink GPU memory when P.Mode entrance is in progress"); return 0; } -#endif mutex_lock(&kctx->jit_evict_lock); list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { @@ -786,9 +753,7 @@ static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s } mutex_unlock(&kctx->jit_evict_lock); -#if MALI_USE_CSF up_read(&kctx->kbdev->csf.mmu_sync_sem); -#endif return freed; } @@ -1494,15 +1459,33 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, in *flags &= ~BASE_MEM_SAME_VA; /* - * Force CPU cached flag. - * - * We can't query the dma-buf exporter to get details about the CPU - * cache attributes of CPU mappings, so we have to assume that the - * buffer may be cached, and call into the exporter for cache - * maintenance, and rely on the exporter to do the right thing when - * handling our calls. + * mgm_get_import_memory_cached_access_permitted() returns true if externally visible + * accesses by the GPU for this dma_buf are allowed to be cached. Set CPU cacheabilty + * flag to match. */ - *flags |= BASE_MEM_CACHED_CPU; + if (kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_cached_access_permitted) { + struct memory_group_manager_import_data mgm_import_data; + + mgm_import_data.type = MEMORY_GROUP_MANAGER_IMPORT_TYPE_DMA_BUF; + mgm_import_data.u.dma_buf = dma_buf; + + if (kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_cached_access_permitted( + kctx->kbdev->mgm_dev, &mgm_import_data)) + *flags |= BASE_MEM_CACHED_CPU; + else + *flags &= ~BASE_MEM_CACHED_CPU; + } else { + /* + * Force CPU cached flag if we do not have the API to query MGM + * + * We can't query the dma-buf exporter to get details about the CPU + * cache attributes of CPU mappings, so we have to assume that the + * buffer may be cached, and call into the exporter for cache + * maintenance, and rely on the exporter to do the right thing when + * handling our calls. + */ + *flags |= BASE_MEM_CACHED_CPU; + } if (*flags & BASE_MEM_IMPORT_SHARED) shared_zone = true; @@ -1825,8 +1808,8 @@ u64 kbase_mem_alias(struct kbase_context *kctx, base_mem_alloc_flags *flags, u64 /* validate and add src handles */ for (i = 0; i < nents; i++) { - if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) { - if (ai[i].handle.basep.handle != BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) + if (ai[i].gpu_va < BASE_MEM_FIRST_FREE_ADDRESS) { + if (ai[i].gpu_va != BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) goto bad_handle; /* unsupported magic handle */ if (!ai[i].length) goto bad_handle; /* must be > 0 */ @@ -1840,7 +1823,7 @@ u64 kbase_mem_alias(struct kbase_context *kctx, base_mem_alloc_flags *flags, u64 struct kbase_mem_phy_alloc *alloc; aliasing_reg = kbase_region_tracker_find_region_base_address( - kctx, (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); + kctx, (ai[i].gpu_va >> PAGE_SHIFT) << PAGE_SHIFT); /* validate found region */ if (kbase_is_region_invalid_or_free(aliasing_reg)) @@ -2734,10 +2717,8 @@ static int kbasep_reg_mmap(struct kbase_context *kctx, struct vm_area_struct *vm kctx->pending_regions[cookie] = NULL; bitmap_set(kctx->cookies, cookie, 1); -#if MALI_USE_CSF if (reg->flags & KBASE_REG_CSF_EVENT) kbase_link_event_mem_page(kctx, reg); -#endif /* * Overwrite the offset with the region start_pfn, so we effectively @@ -2813,7 +2794,6 @@ int kbase_context_mmap(struct kbase_context *const kctx, struct vm_area_struct * err = -EINVAL; goto out_unlock; #endif /* defined(CONFIG_MALI_VECTOR_DUMP) */ -#if MALI_USE_CSF case PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE): kbase_gpu_vm_unlock_with_pmode_sync(kctx); err = kbase_csf_cpu_mmap_user_reg_page(kctx, vma); @@ -2825,7 +2805,6 @@ int kbase_context_mmap(struct kbase_context *const kctx, struct vm_area_struct * mutex_unlock(&kctx->csf.lock); goto out; } -#endif case PFN_DOWN(BASE_MEM_COOKIE_BASE)... PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, &aligned_offset); if (err != 0) @@ -3319,7 +3298,6 @@ static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_ return 0; } -#if MALI_USE_CSF static unsigned long get_queue_doorbell_pfn(struct kbase_device *kbdev, struct kbase_queue *queue) { lockdep_assert_held(&kbdev->csf.reg_lock); @@ -3563,6 +3541,7 @@ static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, struct v get_file(vma->vm_file); /* Also adjust the vm_pgoff */ vma->vm_pgoff = queue->db_file_offset; + kctx->csf.user_io.vma = vma; return 0; @@ -3757,5 +3736,3 @@ static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, struct v return 0; } - -#endif /* MALI_USE_CSF */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_pool.c b/drivers/gpu/arm/valhall/mali_kbase_mem_pool.c index 150236e05e37..3ebbe32ae74c 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_pool.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_pool.c @@ -34,9 +34,8 @@ #include #endif -#define pool_dbg(pool, format, ...) \ - dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, \ - (pool->next_pool) ? "kctx" : "kbdev", kbase_mem_pool_size(pool), \ +#define pool_dbg(pool, format, ...) \ + dev_dbg(pool->kbdev->dev, "%s-pool [%zu/%zu]: " format, "kctx", kbase_mem_pool_size(pool), \ kbase_mem_pool_max_size(pool), ##__VA_ARGS__) #define NOT_DIRTY false @@ -257,14 +256,6 @@ static void kbase_mem_pool_zero_page(struct kbase_mem_pool *pool, struct page *p kbase_mem_pool_sync_page(pool, p); } -static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, struct page *p) -{ - /* Zero page before spilling */ - kbase_mem_pool_zero_page(next_pool, p); - - kbase_mem_pool_add(next_pool, p); -} - struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) { struct page *p; @@ -308,13 +299,15 @@ struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) return p; } +KBASE_EXPORT_TEST_API(kbase_mem_alloc_page); static void enqueue_free_pool_pages_work(struct kbase_mem_pool *pool) { - struct kbase_mem_migrate *mem_migrate = &pool->kbdev->mem_migrate; + if (!pool->order && kbase_is_page_migration_enabled()) { + struct kbase_mem_migrate *mem_migrate = &pool->kbdev->mem_migrate; - if (!pool->order && kbase_is_page_migration_enabled()) queue_work(mem_migrate->free_pages_workq, &mem_migrate->free_pages_work); + } } void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, struct page *p) @@ -345,6 +338,7 @@ void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, struct page *p) pool_dbg(pool, "freed page to kernel\n"); } } +KBASE_EXPORT_TEST_API(kbase_mem_pool_free_page); static size_t kbase_mem_pool_shrink_locked(struct kbase_mem_pool *pool, size_t nr_to_shrink) { @@ -383,10 +377,13 @@ int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow, kbase_mem_pool_lock(pool); - pool->dont_reclaim = true; + pool->reclaim_allowed = false; + for (i = 0; i < nr_to_grow; i++) { if (pool->dying) { - pool->dont_reclaim = false; + if (pool->pool_supports_reclaim) + pool->reclaim_allowed = true; + kbase_mem_pool_shrink_locked(pool, nr_to_grow); kbase_mem_pool_unlock(pool); if (page_owner) @@ -402,9 +399,11 @@ int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow, p = kbase_mem_alloc_page(pool); if (!p) { - kbase_mem_pool_lock(pool); - pool->dont_reclaim = false; - kbase_mem_pool_unlock(pool); + if (pool->pool_supports_reclaim) { + kbase_mem_pool_lock(pool); + pool->reclaim_allowed = true; + kbase_mem_pool_unlock(pool); + } return -ENOMEM; } @@ -412,7 +411,10 @@ int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow, kbase_mem_pool_lock(pool); kbase_mem_pool_add_locked(pool, p); } - pool->dont_reclaim = false; + + if (pool->pool_supports_reclaim) + pool->reclaim_allowed = true; + kbase_mem_pool_unlock(pool); return 0; @@ -443,6 +445,7 @@ void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size) (new_size - cur_size), (grown_size - cur_size)); } } +KBASE_EXPORT_TEST_API(kbase_mem_pool_trim); void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size) { @@ -473,8 +476,12 @@ static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, pool = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_mem_pool, reclaim); + /* Pools not supporting reclaims are not assumed to register reclaim callbacks */ + if (WARN_ON(!pool->pool_supports_reclaim)) + return 0; + kbase_mem_pool_lock(pool); - if (pool->dont_reclaim && !pool->dying) { + if (!pool->reclaim_allowed && !pool->dying) { kbase_mem_pool_unlock(pool); /* Tell shrinker to skip reclaim * even though freeable pages are available @@ -495,10 +502,13 @@ static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, pool = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_mem_pool, reclaim); + if (WARN_ON(!pool->pool_supports_reclaim)) + return SHRINK_STOP; + kbase_mem_pool_lock(pool); - if (pool->dont_reclaim && !pool->dying) { + if (!pool->reclaim_allowed && !pool->dying) { kbase_mem_pool_unlock(pool); - /* Tell shrinker that reclaim can't be made and + /* Tell shrinker that reclaim can't be done, and * do not attempt again for this reclaim context. */ return SHRINK_STOP; @@ -515,9 +525,8 @@ static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, return freed; } -int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool_config *config, - unsigned int order, int group_id, struct kbase_device *kbdev, - struct kbase_mem_pool *next_pool) +static int kbasep_mem_pool_init(struct kbase_mem_pool *pool, size_t max_size, unsigned int order, + int group_id, struct kbase_device *kbdev, bool support_reclaim) { struct shrinker *reclaim; @@ -526,71 +535,73 @@ int kbase_mem_pool_init(struct kbase_mem_pool *pool, const struct kbase_mem_pool } pool->cur_size = 0; - pool->max_size = kbase_mem_pool_config_get_max_size(config); + pool->max_size = max_size; pool->order = order; pool->group_id = group_id; pool->kbdev = kbdev; - pool->next_pool = next_pool; pool->dying = false; + pool->pool_supports_reclaim = support_reclaim; + pool->reclaim_allowed = false; atomic_set(&pool->isolation_in_progress_cnt, 0); spin_lock_init(&pool->pool_lock); INIT_LIST_HEAD(&pool->page_list); - reclaim = KBASE_INIT_RECLAIM(pool, reclaim, "mali-mem-pool"); - if (!reclaim) - return -ENOMEM; - KBASE_SET_RECLAIM(pool, reclaim, reclaim); + if (support_reclaim) { + reclaim = KBASE_INIT_RECLAIM(pool, reclaim, "mali-mem-pool"); + if (!reclaim) + return -ENOMEM; + KBASE_SET_RECLAIM(pool, reclaim, reclaim); - reclaim->count_objects = kbase_mem_pool_reclaim_count_objects; - reclaim->scan_objects = kbase_mem_pool_reclaim_scan_objects; - reclaim->seeks = DEFAULT_SEEKS; - reclaim->batch = 0; + reclaim->count_objects = kbase_mem_pool_reclaim_count_objects; + reclaim->scan_objects = kbase_mem_pool_reclaim_scan_objects; + reclaim->seeks = DEFAULT_SEEKS; + reclaim->batch = 0; - KBASE_REGISTER_SHRINKER(reclaim, "mali-mem-pool", pool); + KBASE_REGISTER_SHRINKER(reclaim, "mali-mem-pool", pool); + } pool_dbg(pool, "initialized\n"); return 0; } + +int kbase_mem_pool_init(struct kbase_mem_pool *pool, size_t max_size, unsigned int order, + int group_id, struct kbase_device *kbdev) +{ + return kbasep_mem_pool_init(pool, max_size, order, group_id, kbdev, true); +} KBASE_EXPORT_TEST_API(kbase_mem_pool_init); +int kbase_mem_pool_init_no_reclaim(struct kbase_mem_pool *pool, size_t max_size, unsigned int order, + int group_id, struct kbase_device *kbdev) +{ + return kbasep_mem_pool_init(pool, max_size, order, group_id, kbdev, false); +} +KBASE_EXPORT_TEST_API(kbase_mem_pool_init_no_reclaim); + void kbase_mem_pool_mark_dying(struct kbase_mem_pool *pool) { kbase_mem_pool_lock(pool); pool->dying = true; kbase_mem_pool_unlock(pool); } +KBASE_EXPORT_TEST_API(kbase_mem_pool_mark_dying); void kbase_mem_pool_term(struct kbase_mem_pool *pool) { - struct kbase_mem_pool *next_pool = pool->next_pool; struct page *p, *tmp; - size_t nr_to_spill = 0; LIST_HEAD(spill_list); LIST_HEAD(free_list); - size_t i; pool_dbg(pool, "terminate()\n"); - KBASE_UNREGISTER_SHRINKER(pool->reclaim); + if (pool->pool_supports_reclaim) + KBASE_UNREGISTER_SHRINKER(pool->reclaim); kbase_mem_pool_lock(pool); pool->max_size = 0; - if (next_pool && !kbase_mem_pool_is_full(next_pool)) { - /* Spill to next pool (may overspill) */ - nr_to_spill = kbase_mem_pool_capacity(next_pool); - nr_to_spill = min(kbase_mem_pool_size(pool), nr_to_spill); - - /* Zero pages first without holding the next_pool lock */ - for (i = 0; i < nr_to_spill; i++) { - p = kbase_mem_pool_remove_locked(pool, SPILL_IN_PROGRESS); - if (p) - list_add(&p->lru, &spill_list); - } - } - while (!kbase_mem_pool_is_empty(pool)) { /* Free remaining pages to kernel */ p = kbase_mem_pool_remove_locked(pool, FREE_IN_PROGRESS); @@ -600,16 +611,6 @@ void kbase_mem_pool_term(struct kbase_mem_pool *pool) kbase_mem_pool_unlock(pool); - if (next_pool && nr_to_spill) { - list_for_each_entry(p, &spill_list, lru) - kbase_mem_pool_zero_page(pool, p); - - /* Add new page list to next_pool */ - kbase_mem_pool_add_list(next_pool, &spill_list, nr_to_spill); - - pool_dbg(pool, "terminate() spilled %zu pages\n", nr_to_spill); - } - list_for_each_entry_safe(p, tmp, &free_list, lru) { list_del_init(&p->lru); kbase_mem_pool_free_page(pool, p); @@ -633,18 +634,12 @@ struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool) { struct page *p; - do { - pool_dbg(pool, "alloc()\n"); - p = kbase_mem_pool_remove(pool, ALLOCATE_IN_PROGRESS); + pool_dbg(pool, "alloc()\n"); + p = kbase_mem_pool_remove(pool, ALLOCATE_IN_PROGRESS); - if (p) - return p; - - pool = pool->next_pool; - } while (pool); - - return NULL; + return p; } +KBASE_EXPORT_TEST_API(kbase_mem_pool_alloc); struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool) { @@ -656,8 +651,6 @@ struct page *kbase_mem_pool_alloc_locked(struct kbase_mem_pool *pool) void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, bool dirty) { - struct kbase_mem_pool *next_pool = pool->next_pool; - pool_dbg(pool, "free()\n"); if (!kbase_mem_pool_is_full(pool)) { @@ -666,9 +659,6 @@ void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, bool dirty kbase_mem_pool_sync_page(pool, p); kbase_mem_pool_add(pool, p); - } else if (next_pool && !kbase_mem_pool_is_full(next_pool)) { - /* Spill to next pool */ - kbase_mem_pool_spill(next_pool, p); } else { /* Free page */ kbase_mem_pool_free_page(pool, p); @@ -676,6 +666,7 @@ void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *p, bool dirty enqueue_free_pool_pages_work(pool); } } +KBASE_EXPORT_TEST_API(kbase_mem_pool_free); void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, bool dirty) { @@ -696,6 +687,7 @@ void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, boo enqueue_free_pool_pages_work(pool); } } +KBASE_EXPORT_TEST_API(kbase_mem_pool_free_locked); int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_small_pages, struct tagged_addr *pages, bool partial_allowed, @@ -735,42 +727,31 @@ int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_small_page } kbase_mem_pool_unlock(pool); - if (i != nr_small_pages && pool->next_pool) { - /* Allocate via next pool */ - err = kbase_mem_pool_alloc_pages(pool->next_pool, nr_small_pages - i, pages + i, - partial_allowed, page_owner); - - if (err < 0) + /* Get any remaining pages from kernel */ + while (i != nr_small_pages) { + if (unlikely(!can_alloc_page(pool, page_owner))) goto err_rollback; - i += (size_t)err; - } else { - /* Get any remaining pages from kernel */ - while (i != nr_small_pages) { - if (unlikely(!can_alloc_page(pool, page_owner))) + p = kbase_mem_alloc_page(pool); + if (!p) { + if (partial_allowed) + goto done; + else goto err_rollback; + } - p = kbase_mem_alloc_page(pool); - if (!p) { - if (partial_allowed) - goto done; - else - goto err_rollback; - } - - if (pool->order) { - uint j; - - pages[i++] = as_tagged_tag(page_to_phys(p), HUGE_PAGE | HUGE_HEAD); - for (j = 1; j < (1u << pool->order); j++) { - phys_addr_t phys; - - phys = page_to_phys(p) + PAGE_SIZE * j; - pages[i++] = as_tagged_tag(phys, HUGE_PAGE); - } - } else { - pages[i++] = as_tagged(page_to_phys(p)); + if (pool->order) { + uint j; + + pages[i++] = as_tagged_tag(page_to_phys(p), HUGE_PAGE | HUGE_HEAD); + for (j = 1; j < (1u << pool->order); j++) { + phys_addr_t phys; + + phys = page_to_phys(p) + PAGE_SIZE * j; + pages[i++] = as_tagged_tag(phys, HUGE_PAGE); } + } else { + pages[i++] = as_tagged(page_to_phys(p)); } } @@ -782,6 +763,7 @@ err_rollback: kbase_mem_pool_free_pages(pool, i, pages, NOT_DIRTY, NOT_RECLAIMED); return err; } +KBASE_EXPORT_TEST_API(kbase_mem_pool_alloc_pages); int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, size_t nr_small_pages, struct tagged_addr *pages) @@ -822,6 +804,7 @@ int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, size_t nr_sma return nr_small_pages; } +KBASE_EXPORT_TEST_API(kbase_mem_pool_alloc_pages_locked); static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, size_t nr_pages, struct tagged_addr *pages, bool zero, bool sync) @@ -902,7 +885,6 @@ static void kbase_mem_pool_add_array_locked(struct kbase_mem_pool *pool, size_t void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, struct tagged_addr *pages, bool dirty, bool reclaimed) { - struct kbase_mem_pool *next_pool = pool->next_pool; struct page *p; size_t nr_to_pool; LIST_HEAD(to_pool_list); @@ -919,15 +901,6 @@ void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, kbase_mem_pool_add_array(pool, nr_to_pool, pages, false, dirty); i += nr_to_pool; - - if (i != nr_pages && next_pool) { - /* Spill to next pool (may overspill) */ - nr_to_pool = kbase_mem_pool_capacity(next_pool); - nr_to_pool = min(nr_pages - i, nr_to_pool); - - kbase_mem_pool_add_array(next_pool, nr_to_pool, pages + i, true, dirty); - i += nr_to_pool; - } } /* Free any remaining pages to kernel */ @@ -952,6 +925,7 @@ void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages, pool_dbg(pool, "free_pages(%zu) done\n", nr_pages); } +KBASE_EXPORT_TEST_API(kbase_mem_pool_free_pages); void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, size_t nr_pages, struct tagged_addr *pages, bool dirty, bool reclaimed) @@ -999,3 +973,4 @@ void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, size_t nr_pag pool_dbg(pool, "free_pages_locked(%zu) done\n", nr_pages); } +KBASE_EXPORT_TEST_API(kbase_mem_pool_free_pages_locked); diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_debugfs.c b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_debugfs.c index b757a1941124..3b89574790ba 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_debugfs.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -165,4 +165,10 @@ void kbase_mem_pool_debugfs_init(struct dentry *parent, struct kbase_context *kc debugfs_create_file("lp_mem_pool_max_size", mode, parent, &kctx->mem_pools.large, &kbase_mem_pool_debugfs_max_size_fops); + + debugfs_create_file("pdg_pool_size", mode, parent, &kctx->pgd_mem_pool, + &kbase_mem_pool_debugfs_fops); + + debugfs_create_file("pgd_mem_pool_max_size", mode, parent, &kctx->pgd_mem_pool, + &kbase_mem_pool_debugfs_max_size_fops); } diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.c b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.c index 1b224f7d0cc0..0c3c90d62995 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -25,36 +25,19 @@ #include -void kbase_mem_pool_group_config_set_max_size(struct kbase_mem_pool_group_config *const configs, - size_t const max_size) -{ - size_t const large_max_size = max_size >> (KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER - - KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER); - int gid; - - for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { - kbase_mem_pool_config_set_max_size(&configs->small[gid], max_size); - - kbase_mem_pool_config_set_max_size(&configs->large[gid], large_max_size); - } -} - int kbase_mem_pool_group_init(struct kbase_mem_pool_group *const mem_pools, - struct kbase_device *const kbdev, - const struct kbase_mem_pool_group_config *const configs, - struct kbase_mem_pool_group *next_pools) + struct kbase_device *const kbdev, size_t small_max_size, + size_t large_max_size, struct kbase_mem_pool_group *next_pools) { int gid, err = 0; for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { - err = kbase_mem_pool_init(&mem_pools->small[gid], &configs->small[gid], - KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, gid, kbdev, - next_pools ? &next_pools->small[gid] : NULL); + err = kbase_mem_pool_init(&mem_pools->small[gid], small_max_size, + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, gid, kbdev); if (!err) { - err = kbase_mem_pool_init(&mem_pools->large[gid], &configs->large[gid], - KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, gid, kbdev, - next_pools ? &next_pools->large[gid] : NULL); + err = kbase_mem_pool_init(&mem_pools->large[gid], large_max_size, + KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER, gid, kbdev); if (err) kbase_mem_pool_term(&mem_pools->small[gid]); } diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.h b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.h index cb69eab061ac..b32c71e3db4a 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.h +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_pool_group.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -24,53 +24,15 @@ #include -/** - * kbase_mem_pool_group_select() - Select the memory pool to use. - * - * @kbdev: Device pointer. - * @mem_group_id: Physical memory group ID to use. - * @is_small_page: Flag used to select between the small and - * large memory pool. - * - * Return: A pointer to the selected memory pool. - */ -static inline struct kbase_mem_pool * -kbase_mem_pool_group_select(struct kbase_device *kbdev, u32 mem_group_id, bool is_small_page) -{ - if (WARN_ON(unlikely(kbdev == NULL))) - return NULL; - - WARN_ON(mem_group_id > BASE_MEM_GROUP_COUNT); - - if (is_small_page) - return &kbdev->mem_pools.small[mem_group_id]; - - return &kbdev->mem_pools.large[mem_group_id]; -} - -/** - * kbase_mem_pool_group_config_set_max_size - Set the initial configuration for - * a set of memory pools - * - * @configs: Initial configuration for the set of memory pools - * @max_size: Maximum number of free small pages each pool can hold - * - * This function sets the initial configuration for every memory pool so that - * the maximum amount of free memory that each pool can hold is identical. - * The equivalent number of 2 MiB pages is calculated automatically for the - * purpose of configuring the large page pools. - */ -void kbase_mem_pool_group_config_set_max_size(struct kbase_mem_pool_group_config *configs, - size_t max_size); - /** * kbase_mem_pool_group_init - Initialize a set of memory pools * - * @mem_pools: Set of memory pools to initialize - * @kbdev: Kbase device where memory is used - * @configs: Initial configuration for the set of memory pools - * @next_pools: Set of memory pools from which to allocate memory if there - * is no free memory in one of the @mem_pools + * @mem_pools: Set of memory pools to initialize + * @kbdev: Kbase device where memory is used + * @small_max_size: Maximum size for small memory pools + * @large_max_size: Maximum size for large memory pools + * @next_pools: Set of memory pools from which to allocate memory if there + * is no free memory in one of the @mem_pools * * Initializes a complete set of physical memory pools. Memory pools are used to * allow efficient reallocation of previously-freed physical pages. A pair of @@ -87,7 +49,7 @@ void kbase_mem_pool_group_config_set_max_size(struct kbase_mem_pool_group_config * Return: 0 on success, otherwise a negative error code */ int kbase_mem_pool_group_init(struct kbase_mem_pool_group *mem_pools, struct kbase_device *kbdev, - const struct kbase_mem_pool_group_config *configs, + size_t small_max_size, size_t large_max_size, struct kbase_mem_pool_group *next_pools); /** diff --git a/drivers/gpu/arm/valhall/mali_kbase_pbha.c b/drivers/gpu/arm/valhall/mali_kbase_pbha.c index c201163b5424..b79d419e24de 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_pbha.c +++ b/drivers/gpu/arm/valhall/mali_kbase_pbha.c @@ -24,9 +24,7 @@ #include #include -#if MALI_USE_CSF #define DTB_SET_SIZE 2 -#endif static bool read_setting_valid(unsigned int prod_model, unsigned int id, unsigned int read_setting) { @@ -164,7 +162,6 @@ int kbase_pbha_record_settings(struct kbase_device *kbdev, bool runtime, unsigne unsigned int const sysc_alloc_num = id / sizeof(u32); u32 modified_reg; -#if MALI_USE_CSF if (runtime) { uint i; @@ -175,9 +172,6 @@ int kbase_pbha_record_settings(struct kbase_device *kbdev, bool runtime, unsigne kbase_reg_read32(kbdev, GPU_SYSC_ALLOC_OFFSET(i)); kbase_pm_context_idle(kbdev); } -#else - CSTD_UNUSED(runtime); -#endif /* MALI_USE_CSF */ modified_reg = kbdev->sysc_alloc[sysc_alloc_num]; @@ -224,7 +218,6 @@ int kbase_pbha_record_settings(struct kbase_device *kbdev, bool runtime, unsigne void kbase_pbha_write_settings(struct kbase_device *kbdev) { -#if MALI_USE_CSF if (kbasep_pbha_supported(kbdev)) { uint i; @@ -249,12 +242,8 @@ void kbase_pbha_write_settings(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_SYSC_PBHA_OVERRIDE_OFFSET(reg_index), pbha_override_val); } -#else - CSTD_UNUSED(kbdev); -#endif /* MALI_USE_CSF */ } -#if MALI_USE_CSF static int kbase_pbha_read_int_id_override_property(struct kbase_device *kbdev, const struct device_node *pbha_node) { @@ -376,11 +365,9 @@ static int kbase_pbha_read_mma_wa_id_property(struct kbase_device *kbdev, kbdev->mma_wa_id = mma_wa_id; return 0; } -#endif /* MALI_USE_CSF */ int kbase_pbha_read_dtb(struct kbase_device *kbdev) { -#if MALI_USE_CSF const struct device_node *pbha_node; int err; @@ -404,7 +391,4 @@ int kbase_pbha_read_dtb(struct kbase_device *kbdev) err = kbase_pbha_read_mma_wa_id_property(kbdev, pbha_node); return err; -#else - return 0; -#endif } diff --git a/drivers/gpu/arm/valhall/mali_kbase_pbha_debugfs.c b/drivers/gpu/arm/valhall/mali_kbase_pbha_debugfs.c index 81f2df5ea977..8341d664dd34 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_pbha_debugfs.c +++ b/drivers/gpu/arm/valhall/mali_kbase_pbha_debugfs.c @@ -25,9 +25,7 @@ #include #include -#if MALI_USE_CSF #include "backend/gpu/mali_kbase_pm_internal.h" -#endif static int int_id_overrides_show(struct seq_file *sfile, void *data) { @@ -43,11 +41,7 @@ static int int_id_overrides_show(struct seq_file *sfile, void *data) for (i = 0; i < GPU_SYSC_ALLOC_COUNT; ++i) { uint j; -#if MALI_USE_CSF u32 reg = kbase_reg_read32(kbdev, GPU_SYSC_ALLOC_OFFSET(i)); -#else /* MALI_USE_CSF */ - u32 reg = 0; -#endif /* MALI_USE_CSF */ for (j = 0; j < sizeof(u32); ++j) { u8 r_val = 0; @@ -120,7 +114,6 @@ static int int_id_overrides_open(struct inode *in, struct file *file) return single_open(file, int_id_overrides_show, in->i_private); } -#if MALI_USE_CSF /** * propagate_bits_show - Read PBHA bits from L2_CONFIG out to debugfs. * @@ -207,7 +200,6 @@ static const struct file_operations pbha_propagate_bits_fops = { .llseek = seq_lseek, .release = single_release, }; -#endif /* MALI_USE_CSF */ static const struct file_operations pbha_int_id_overrides_fops = { .owner = THIS_MODULE, @@ -233,10 +225,8 @@ void kbase_pbha_debugfs_init(struct kbase_device *kbdev) debugfs_create_file("int_id_overrides", mode, debugfs_pbha_dir, kbdev, &pbha_int_id_overrides_fops); -#if MALI_USE_CSF if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) debugfs_create_file("propagate_bits", mode, debugfs_pbha_dir, kbdev, &pbha_propagate_bits_fops); -#endif /* MALI_USE_CSF */ } } diff --git a/drivers/gpu/arm/valhall/mali_kbase_pm.c b/drivers/gpu/arm/valhall/mali_kbase_pm.c index e7e70b5389d8..d13cbbe48458 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_pm.c +++ b/drivers/gpu/arm/valhall/mali_kbase_pm.c @@ -35,16 +35,6 @@ #include -int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) -{ - return kbase_hwaccess_pm_powerup(kbdev, flags); -} - -void kbase_pm_halt(struct kbase_device *kbdev) -{ - kbase_hwaccess_pm_halt(kbdev); -} - void kbase_pm_context_active(struct kbase_device *kbdev) { (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE); @@ -62,13 +52,11 @@ kbasep_pm_context_active_handle_suspend_locked(struct kbase_device *kbdev, current->pid); lockdep_assert_held(&kbdev->pm.lock); -#if MALI_USE_CSF /* Check scheduler lock */ if (sched_lock_held) lockdep_assert_held(&kbdev->csf.scheduler.lock); else kbase_lockdep_assert_not_held(&kbdev->csf.scheduler.lock); -#endif /* If there is an Arbiter, wait for Arbiter to grant GPU back to KBase * so suspend request can be handled. @@ -170,40 +158,20 @@ static void reenable_hwcnt_on_resume(struct kbase_device *kbdev) unsigned long flags; /* Re-enable GPU hardware counters */ -#if MALI_USE_CSF - kbase_csf_scheduler_spin_lock(kbdev, &flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - kbase_csf_scheduler_spin_unlock(kbdev, flags); -#else - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#endif + if (kbdev->csf.firmware_hctl_core_pwr) { + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + } /* Resume HW counters intermediaries. */ -#if MALI_USE_CSF if (kbdev->csf.firmware_inited) -#endif - { kbase_kinstr_prfcnt_resume(kbdev->kinstr_prfcnt_ctx); - } } static void resume_job_scheduling(struct kbase_device *kbdev) { -#if !MALI_USE_CSF - /* Resume any blocked atoms (which may cause contexts to be scheduled in - * and dependent atoms to run) - */ - kbase_resume_suspended_soft_jobs(kbdev); - - /* Resume the Job Scheduler and associated components, and start running - * atoms - */ - kbasep_js_resume(kbdev); -#else kbase_csf_scheduler_pm_resume(kbdev); -#endif } int kbase_pm_driver_suspend(struct kbase_device *kbdev) @@ -214,17 +182,14 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) /* Suspend HW counter intermediaries. This blocks until workers and timers * are no longer running. */ -#if MALI_USE_CSF if (kbdev->csf.firmware_inited) -#endif - { kbase_kinstr_prfcnt_suspend(kbdev->kinstr_prfcnt_ctx); - } /* Disable GPU hardware counters. * This call will block until counters are disabled. */ - kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); + if (kbdev->csf.firmware_hctl_core_pwr) + kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); mutex_lock(&kbdev->pm.lock); if (WARN_ON(kbase_pm_is_suspending(kbdev))) { @@ -238,20 +203,9 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) if (kbase_has_arbiter(kbdev)) { unsigned long flags; -#if MALI_USE_CSF spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_disjoint_state_up(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#else - unsigned int i; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbdev->js_data.runpool_irq.submit_allowed = 0; - kbase_disjoint_state_up(kbdev); - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) - kbase_job_slot_softstop(kbdev, i, NULL); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#endif } /* From now on, the active count will drop towards zero. Sometimes, @@ -260,15 +214,8 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) * all pm references */ -#if !MALI_USE_CSF - /* Suspend job scheduler and associated components, so that it releases all - * the PM active count references - */ - kbasep_js_suspend(kbdev); -#else if (kbase_csf_scheduler_pm_suspend(kbdev)) goto exit; -#endif scheduling_suspended = true; @@ -281,7 +228,6 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0); dev_dbg(kbdev->dev, ">wait_event - waiting done\n"); -#if MALI_USE_CSF /* At this point, any kbase context termination should either have run to * completion and any further context termination can only begin after * the system resumes. Therefore, it is now safe to skip taking the context @@ -289,7 +235,6 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) */ if (kbase_csf_kcpu_queue_halt_timers(kbdev)) goto exit; -#endif timers_halted = true; @@ -310,21 +255,17 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) mutex_unlock(&kbdev->pm.arb_vm_state->vm_state_lock); } -#if MALI_USE_CSF kbase_backend_invalidate_gpu_timestamp_offset(kbdev); -#endif return 0; exit: if (timers_halted) { -#if MALI_USE_CSF /* Resume the timers in case of suspend failure. But that needs to * be done before clearing the 'pm.suspending' flag so as to keep the * context termination blocked. */ kbase_csf_kcpu_queue_resume_timers(kbdev); -#endif } mutex_lock(&kbdev->pm.lock); @@ -358,9 +299,7 @@ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) resume_job_scheduling(kbdev); -#if MALI_USE_CSF kbase_csf_kcpu_queue_resume_timers(kbdev); -#endif /* Matching idle call, to power off the GPU/cores if we didn't actually * need it and the policy doesn't want it on diff --git a/drivers/gpu/arm/valhall/mali_kbase_pm.h b/drivers/gpu/arm/valhall/mali_kbase_pm.h index f176f8cdfc31..e1c5388a7d54 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_pm.h +++ b/drivers/gpu/arm/valhall/mali_kbase_pm.h @@ -52,29 +52,6 @@ struct kbase_device; */ int kbase_pm_init(struct kbase_device *kbdev); -/** - * kbase_pm_powerup - Power up GPU after all modules have been initialized - * and interrupt handlers installed. - * - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * @flags: Flags to pass on to kbase_pm_init_hw - * - * Return: 0 if powerup was successful. - */ -int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags); - -/** - * kbase_pm_halt - Halt the power management framework. - * - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Should ensure that no new interrupts are generated, - * but allow any currently running interrupt handlers to complete successfully. - * The GPU is forced off by the time this function returns, regardless of - * whether or not the active power policy asks for the GPU to be powered off. - */ -void kbase_pm_halt(struct kbase_device *kbdev); - /** * kbase_pm_term - Terminate the power management framework. * diff --git a/drivers/gpu/arm/valhall/mali_kbase_reg_track.c b/drivers/gpu/arm/valhall/mali_kbase_reg_track.c index 2d9ae789f7f4..9c26e33be18d 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_reg_track.c +++ b/drivers/gpu/arm/valhall/mali_kbase_reg_track.c @@ -42,14 +42,12 @@ char *kbase_reg_zone_get_name(enum kbase_memory_zone zone) return "CUSTOM_VA"; case EXEC_VA_ZONE: return "EXEC_VA"; -#if MALI_USE_CSF case MCU_SHARED_ZONE: return "MCU_SHARED"; case EXEC_FIXED_VA_ZONE: return "EXEC_FIXED_VA"; case FIXED_VA_ZONE: return "FIXED_VA"; -#endif default: return NULL; } @@ -345,7 +343,7 @@ kbase_region_tracker_find_region_meeting_reqs(struct kbase_va_region *reg_reqs, void kbase_remove_va_region(struct kbase_device *kbdev, struct kbase_va_region *reg) { struct rb_node *rbprev; - struct kbase_reg_zone *zone = container_of(reg->rbtree, struct kbase_reg_zone, reg_rbtree); + struct kbase_reg_zone *zone; struct kbase_va_region *prev = NULL; struct rb_node *rbnext; struct kbase_va_region *next = NULL; @@ -355,6 +353,10 @@ void kbase_remove_va_region(struct kbase_device *kbdev, struct kbase_va_region * int merged_front = 0; int merged_back = 0; + if (WARN_ON(reg == NULL)) + return; + + zone = container_of(reg->rbtree, struct kbase_reg_zone, reg_rbtree); reg_rbtree = reg->rbtree; if (WARN_ON(RB_EMPTY_ROOT(reg_rbtree))) @@ -566,14 +568,10 @@ int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, * Also, executable allocations from EXEC_VA don't need the special * alignment. */ -#if MALI_USE_CSF /* The same is also true for the EXEC_FIXED_VA zone. */ -#endif if (!(reg->flags & KBASE_REG_GPU_NX) && !addr && -#if MALI_USE_CSF ((kbase_bits_to_zone(reg->flags)) != EXEC_FIXED_VA_ZONE) && -#endif ((kbase_bits_to_zone(reg->flags)) != EXEC_VA_ZONE)) { if (cpu_va_bits > gpu_pc_bits) { align = max(align, (size_t)((1ULL << gpu_pc_bits) >> PAGE_SHIFT)); @@ -665,16 +663,6 @@ int kbase_add_va_region_rbtree(struct kbase_device *kbdev, struct kbase_va_regio size_t align_offset = align; size_t align_mask = align - 1; -#if !MALI_USE_CSF - if ((reg->flags & KBASE_REG_TILER_ALIGN_TOP)) { - WARN(align > 1, - "%s with align %lx might not be honored for KBASE_REG_TILER_ALIGN_TOP memory", - __func__, (unsigned long)align); - align_mask = reg->extension - 1; - align_offset = reg->extension - reg->initial_commit; - } -#endif /* !MALI_USE_CSF */ - tmp = kbase_region_tracker_find_region_meeting_reqs(reg, nr_pages, align_offset, align_mask, &start_pfn); if (tmp) { @@ -768,7 +756,6 @@ static int kbase_reg_zone_same_va_init(struct kbase_context *kctx, u64 gpu_va_li lockdep_assert_held(&kctx->reg_lock); -#if MALI_USE_CSF if ((base_pfn + nr_pages) > KBASE_REG_ZONE_EXEC_VA_BASE_64) { /* Depending on how the kernel is configured, it's possible (eg on aarch64) for * same_va_bits to reach 48 bits. Cap same_va_pages so that the same_va zone @@ -776,7 +763,6 @@ static int kbase_reg_zone_same_va_init(struct kbase_context *kctx, u64 gpu_va_li */ nr_pages = KBASE_REG_ZONE_EXEC_VA_BASE_64 - base_pfn; } -#endif err = kbase_reg_zone_init(kctx->kbdev, zone, SAME_VA_ZONE, base_pfn, nr_pages); if (err) return -ENOMEM; @@ -841,17 +827,9 @@ static inline u64 kbase_get_exec_va_zone_base(struct kbase_context *kctx) { u64 base_pfn; -#if MALI_USE_CSF base_pfn = KBASE_REG_ZONE_EXEC_VA_BASE_64; if (kbase_ctx_compat_mode(kctx)) base_pfn = KBASE_REG_ZONE_EXEC_VA_BASE_32; -#else - CSTD_UNUSED(kctx); - /* EXEC_VA zone's codepaths are slightly easier when its base_pfn is - * initially U64_MAX - */ - base_pfn = U64_MAX; -#endif return base_pfn; } @@ -864,10 +842,6 @@ static inline int kbase_reg_zone_exec_va_init(struct kbase_context *kctx, u64 gp CSTD_UNUSED(gpu_va_limit); -#if !MALI_USE_CSF - nr_pages = 0; -#endif - return kbase_reg_zone_init(kctx->kbdev, zone, EXEC_VA_ZONE, base_pfn, nr_pages); } @@ -878,7 +852,6 @@ static void kbase_reg_zone_exec_va_term(struct kbase_context *kctx) kbase_reg_zone_term(zone); } -#if MALI_USE_CSF static inline u64 kbase_get_exec_fixed_va_zone_base(struct kbase_context *kctx) { return kbase_get_exec_va_zone_base(kctx) + KBASE_REG_ZONE_EXEC_VA_SIZE; @@ -931,7 +904,6 @@ static void kbase_reg_zone_fixed_va_term(struct kbase_context *kctx) kbase_reg_zone_term(zone); } -#endif typedef int kbase_memory_zone_init(struct kbase_context *kctx, u64 gpu_va_limit); typedef void kbase_memory_zone_term(struct kbase_context *kctx); @@ -949,13 +921,11 @@ static const struct kbase_memory_zone_init_meta zones_init[] = { "Could not initialize CUSTOM_VA zone" }, [EXEC_VA_ZONE] = { kbase_reg_zone_exec_va_init, kbase_reg_zone_exec_va_term, "Could not initialize EXEC_VA zone" }, -#if MALI_USE_CSF [EXEC_FIXED_VA_ZONE] = { kbase_reg_zone_exec_fixed_va_init, kbase_reg_zone_exec_fixed_va_term, "Could not initialize EXEC_FIXED_VA zone" }, [FIXED_VA_ZONE] = { kbase_reg_zone_fixed_va_init, kbase_reg_zone_fixed_va_term, "Could not initialize FIXED_VA zone" }, -#endif }; int kbase_region_tracker_init(struct kbase_context *kctx) @@ -975,9 +945,7 @@ int kbase_region_tracker_init(struct kbase_context *kctx) goto term; } } -#if MALI_USE_CSF INIT_LIST_HEAD(&kctx->csf.event_pages_head); -#endif kctx->jit_va = false; kbase_gpu_vm_unlock(kctx); @@ -1217,16 +1185,6 @@ KBASE_EXPORT_TEST_API(kbase_region_tracker_init_jit); int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages) { -#if !MALI_USE_CSF - struct kbase_reg_zone *exec_va_zone; - struct kbase_reg_zone *target_zone; - struct kbase_va_region *target_reg; - u64 target_zone_base_addr; - enum kbase_memory_zone target_zone_id; - u64 exec_va_start; - int err; -#endif - /* The EXEC_VA zone shall be created by making space either: * - for 64-bit clients, at the end of the process's address space * - for 32-bit clients, in the CUSTOM zone @@ -1239,85 +1197,14 @@ int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages if (exec_va_pages == 0 || exec_va_pages > KBASE_REG_ZONE_EXEC_VA_MAX_PAGES) return -EINVAL; -#if MALI_USE_CSF /* For CSF GPUs we now setup the EXEC_VA zone during initialization, * so this request is a null-op. */ CSTD_UNUSED(kctx); return 0; -#else - kbase_gpu_vm_lock(kctx); - - /* Verify that we've not already created a EXEC_VA zone, and that the - * EXEC_VA zone must come before JIT's CUSTOM_VA. - */ - if (kbase_has_exec_va_zone_locked(kctx) || kctx->jit_va) { - err = -EPERM; - goto exit_unlock; - } - - if (exec_va_pages > kctx->gpu_va_end) { - err = -ENOMEM; - goto exit_unlock; - } - - /* Verify no allocations have already been made */ - if (kbase_region_tracker_has_allocs(kctx)) { - err = -ENOMEM; - goto exit_unlock; - } - - if (kbase_ctx_compat_mode(kctx)) { - /* 32-bit client: take from CUSTOM_VA zone */ - target_zone_id = CUSTOM_VA_ZONE; - } else { - /* 64-bit client: take from SAME_VA zone */ - target_zone_id = SAME_VA_ZONE; - } - - target_zone = kbase_ctx_reg_zone_get(kctx, target_zone_id); - target_zone_base_addr = target_zone->base_pfn << PAGE_SHIFT; - - target_reg = kbase_region_tracker_find_region_base_address(kctx, target_zone_base_addr); - if (WARN(!target_reg, - "Already found a free region at the start of every zone, but now cannot find any region for zone base 0x%.16llx zone %s", - (unsigned long long)target_zone_base_addr, - kbase_reg_zone_get_name(target_zone_id))) { - err = -ENOMEM; - goto exit_unlock; - } - /* kbase_region_tracker_has_allocs() above has already ensured that all - * of the zones have no allocs, so no need to check that again on - * target_reg - */ - WARN_ON((!(target_reg->flags & KBASE_REG_FREE)) || - target_reg->nr_pages != target_zone->va_size_pages); - - if (target_reg->nr_pages <= exec_va_pages || target_zone->va_size_pages <= exec_va_pages) { - err = -ENOMEM; - goto exit_unlock; - } - - /* Taken from the end of the target zone */ - exec_va_start = kbase_reg_zone_end_pfn(target_zone) - exec_va_pages; - exec_va_zone = kbase_ctx_reg_zone_get(kctx, EXEC_VA_ZONE); - if (kbase_reg_zone_init(kctx->kbdev, exec_va_zone, EXEC_VA_ZONE, exec_va_start, - exec_va_pages)) - return -ENOMEM; - - /* Update target zone and corresponding region */ - target_reg->nr_pages -= exec_va_pages; - target_zone->va_size_pages -= exec_va_pages; - err = 0; - -exit_unlock: - kbase_gpu_vm_unlock(kctx); - return err; -#endif /* MALI_USE_CSF */ } KBASE_EXPORT_TEST_API(kbase_region_tracker_init_exec); -#if MALI_USE_CSF void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev) { kbase_reg_zone_term(&kbdev->csf.mcu_shared_zone); @@ -1328,7 +1215,6 @@ int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev) return kbase_reg_zone_init(kbdev, &kbdev->csf.mcu_shared_zone, MCU_SHARED_ZONE, KBASE_REG_ZONE_MCU_SHARED_BASE, MCU_SHARED_ZONE_SIZE); } -#endif /** * kbase_alloc_free_region - Allocate a free region object. @@ -1410,12 +1296,10 @@ KBASE_EXPORT_TEST_API(kbase_ctx_alloc_free_region); */ void kbase_free_alloced_region(struct kbase_va_region *reg) { -#if MALI_USE_CSF if (kbase_bits_to_zone(reg->flags) == MCU_SHARED_ZONE) { kfree(reg); return; } -#endif if (!(reg->flags & KBASE_REG_FREE)) { struct kbase_context *kctx = kbase_reg_to_kctx(reg); @@ -1427,7 +1311,6 @@ void kbase_free_alloced_region(struct kbase_va_region *reg) dev_dbg(kctx->kbdev->dev, "Freeing memory region %pK\n of zone %s", (void *)reg, kbase_reg_zone_get_name(kbase_bits_to_zone(reg->flags))); -#if MALI_USE_CSF if (reg->flags & KBASE_REG_CSF_EVENT) /* * This should not be reachable if called from 'mcu_shared' functions @@ -1437,7 +1320,6 @@ void kbase_free_alloced_region(struct kbase_va_region *reg) */ kbase_unlink_event_mem_page(kctx, reg); -#endif mutex_lock(&kctx->jit_evict_lock); diff --git a/drivers/gpu/arm/valhall/mali_kbase_reg_track.h b/drivers/gpu/arm/valhall/mali_kbase_reg_track.h index d29999fdcb9a..376951a9d569 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_reg_track.h +++ b/drivers/gpu/arm/valhall/mali_kbase_reg_track.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -33,22 +33,13 @@ struct kbase_va_region; struct kbase_device; struct kmem_cache; -#if MALI_USE_CSF /* Space for 8 different zones */ #define KBASE_REG_ZONE_BITS 3 -#else -/* Space for 4 different zones */ -#define KBASE_REG_ZONE_BITS 2 -#endif /** * KBASE_REG_ZONE_MAX - Maximum number of GPU memory region zones */ -#if MALI_USE_CSF #define KBASE_REG_ZONE_MAX 6ul -#else -#define KBASE_REG_ZONE_MAX 4ul -#endif /* The bits 11-13 (inclusive) of the kbase_va_region flag are reserved * for information about the zone in which it was allocated. @@ -62,22 +53,10 @@ struct kmem_cache; #define KBASE_REG_ZONE_CUSTOM_VA_BASE (0x100000000ULL >> PAGE_SHIFT) -#if MALI_USE_CSF /* only used with 32-bit clients */ /* On a 32bit platform, custom VA should be wired from 4GB to 2^(43). */ #define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 43) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) -#else -/* only used with 32-bit clients */ -/* On a 32bit platform, custom VA should be wired from 4GB to the VA limit of the - * GPU. Unfortunately, the Linux mmap() interface limits us to 2^32 pages (2^44 - * bytes, see mmap64 man page for reference). So we put the default limit to the - * maximum possible on Linux and shrink it down, if required by the GPU, during - * initialization. - */ -#define KBASE_REG_ZONE_CUSTOM_VA_SIZE (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE) -/* end 32-bit clients only */ -#endif /* The starting address and size of the GPU-executable zone are dynamic * and depend on the platform and the number of pages requested by the @@ -86,7 +65,6 @@ struct kmem_cache; #define KBASE_REG_ZONE_EXEC_VA_MAX_PAGES ((1ULL << 32) >> PAGE_SHIFT) /* 4 GB */ #define KBASE_REG_ZONE_EXEC_VA_SIZE KBASE_REG_ZONE_EXEC_VA_MAX_PAGES -#if MALI_USE_CSF #define KBASE_REG_ZONE_MCU_SHARED_BASE (0x04000000ULL >> PAGE_SHIFT) #define MCU_SHARED_ZONE_SIZE (((0x08000000ULL) >> PAGE_SHIFT) - KBASE_REG_ZONE_MCU_SHARED_BASE) @@ -109,8 +87,6 @@ struct kmem_cache; #define KBASE_REG_ZONE_FIXED_VA_END_64 ((1ULL << 48) >> PAGE_SHIFT) #define KBASE_REG_ZONE_FIXED_VA_END_32 ((1ULL << 44) >> PAGE_SHIFT) -#endif - /** * enum kbase_memory_zone - Kbase memory zone identifier * @SAME_VA_ZONE: Memory zone for allocations where the GPU and CPU VA coincide. @@ -142,17 +118,11 @@ enum kbase_memory_zone { SAME_VA_ZONE, CUSTOM_VA_ZONE, EXEC_VA_ZONE, -#if IS_ENABLED(MALI_USE_CSF) EXEC_FIXED_VA_ZONE, FIXED_VA_ZONE, MCU_SHARED_ZONE, -#endif MEMORY_ZONE_MAX, -#if IS_ENABLED(MALI_USE_CSF) CONTEXT_ZONE_MAX = FIXED_VA_ZONE + 1 -#else - CONTEXT_ZONE_MAX = EXEC_VA_ZONE + 1 -#endif }; /** @@ -196,7 +166,7 @@ unsigned long kbase_zone_to_bits(enum kbase_memory_zone zone); enum kbase_memory_zone kbase_bits_to_zone(unsigned long zone_bits); /** - * kbase_mem_zone_get_name - Get the string name for a given memory zone + * kbase_reg_zone_get_name - Get the string name for a given memory zone * @zone: Memory zone identifier * * Return: string for valid memory zone, NULL otherwise @@ -212,11 +182,7 @@ char *kbase_reg_zone_get_name(enum kbase_memory_zone zone); */ static inline bool kbase_is_ctx_reg_zone(enum kbase_memory_zone zone) { -#if MALI_USE_CSF return !(zone == MCU_SHARED_ZONE); -#else - return true; -#endif } /** diff --git a/drivers/gpu/arm/valhall/mali_kbase_softjobs.c b/drivers/gpu/arm/valhall/mali_kbase_softjobs.c index 9f060f42d8dd..df66b3b81609 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_softjobs.c +++ b/drivers/gpu/arm/valhall/mali_kbase_softjobs.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -41,830 +40,6 @@ #include #include -#if !MALI_USE_CSF - -/** - * struct kbase_external_resource_list - Structure which describes a list of external - * resources. This structure is used for the processing of - * EXT_RES_MAP and EXT_RES_UNMAP softjobs, instead of ioctl - * structure 'base_external_resource_list'. This is done to - * avoid UBSAN falsely detecting that an out of bound access - * is going to be made for ext_res[1] array, defined inside - * 'base_external_resource_list', when number of external - * resources to be processed are more than 1. - * - * @count: The number of external resources. - * @ext_res: Pointer to an array of external resources. - */ -struct kbase_external_resource_list { - u64 count; - struct base_external_resource *ext_res; -}; - -/** - * DOC: This file implements the logic behind software only jobs that are - * executed within the driver rather than being handed over to the GPU. - */ - -static void kbasep_add_waiting_soft_job(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - unsigned long lflags; - - spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - list_add_tail(&katom->queue, &kctx->waiting_soft_jobs); - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -} - -void kbasep_remove_waiting_soft_job(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - unsigned long lflags; - - spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - list_del(&katom->queue); - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -} - -static void kbasep_add_waiting_with_timeout(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - - /* Record the start time of this atom so we could cancel it at - * the right time. - */ - katom->start_timestamp = ktime_get_raw(); - - /* Add the atom to the waiting list before the timer is - * (re)started to make sure that it gets processed. - */ - kbasep_add_waiting_soft_job(katom); - - /* Schedule timeout of this atom after a period if it is not active */ - if (!timer_pending(&kctx->soft_job_timeout)) { - unsigned int timeout_ms = - (unsigned int)atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); - mod_timer(&kctx->soft_job_timeout, jiffies + msecs_to_jiffies(timeout_ms)); - } -} - -static int kbasep_read_soft_event_status(struct kbase_context *kctx, u64 evt, unsigned char *status) -{ - unsigned char *mapped_evt; - struct kbase_vmap_struct map; - - mapped_evt = kbase_vmap_prot(kctx, evt, sizeof(*mapped_evt), KBASE_REG_CPU_RD, &map); - if (!mapped_evt) - return -EFAULT; - - *status = *mapped_evt; - - kbase_vunmap(kctx, &map); - - return 0; -} - -static int kbasep_write_soft_event_status(struct kbase_context *kctx, u64 evt, - unsigned char new_status) -{ - unsigned char *mapped_evt; - struct kbase_vmap_struct map; - - if ((new_status != BASE_JD_SOFT_EVENT_SET) && (new_status != BASE_JD_SOFT_EVENT_RESET)) - return -EINVAL; - - mapped_evt = kbase_vmap_prot(kctx, evt, sizeof(*mapped_evt), KBASE_REG_CPU_WR, &map); - if (!mapped_evt) - return -EFAULT; - - *mapped_evt = new_status; - - kbase_vunmap(kctx, &map); - - return 0; -} - -static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) -{ - struct kbase_vmap_struct map; - void *user_result; - struct timespec64 ts; - struct base_dump_cpu_gpu_counters data; - u64 system_time = 0ULL; - u64 cycle_counter; - u64 jc = katom->jc; - struct kbase_context *kctx = katom->kctx; - int pm_active_err; - - memset(&data, 0, sizeof(data)); - - /* Take the PM active reference as late as possible - otherwise, it could - * delay suspend until we process the atom (which may be at the end of a - * long chain of dependencies - */ - if (kbase_has_arbiter(kctx->kbdev)) - atomic_inc(&kctx->kbdev->pm.gpu_users_waiting); - pm_active_err = kbase_pm_context_active_handle_suspend( - kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); - if (pm_active_err) { - struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; - - /* We're suspended - queue this on the list of suspended jobs - * Use dep_item[1], because dep_item[0] was previously in use - * for 'waiting_soft_jobs'. - */ - mutex_lock(&js_devdata->runpool_mutex); - list_add_tail(&katom->dep_item[1], &js_devdata->suspended_soft_jobs_list); - mutex_unlock(&js_devdata->runpool_mutex); - - /* Also adding this to the list of waiting soft job */ - kbasep_add_waiting_soft_job(katom); - - return pm_active_err; - } else if (kbase_has_arbiter(kctx->kbdev)) - atomic_dec(&kctx->kbdev->pm.gpu_users_waiting); - - kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, &ts); - - kbase_pm_context_idle(kctx->kbdev); - - data.sec = (__u64)ts.tv_sec; - data.usec = ts.tv_nsec / 1000; - data.system_time = system_time; - data.cycle_counter = cycle_counter; - - /* Assume this atom will be cancelled until we know otherwise */ - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - /* GPU_WR access is checked on the range for returning the result to - * userspace for the following reasons: - * - security, this is currently how imported user bufs are checked. - * - userspace ddk guaranteed to assume region was mapped as GPU_WR - */ - user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); - if (!user_result) - return 0; - - memcpy(user_result, &data, sizeof(data)); - - kbase_vunmap(kctx, &map); - - /* Atom was fine - mark it as done */ - katom->event_code = BASE_JD_EVENT_DONE; - - return 0; -} - -#if IS_ENABLED(CONFIG_SYNC_FILE) -/* Called by the explicit fence mechanism when a fence wait has completed */ -void kbase_soft_event_wait_callback(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - - mutex_lock(&kctx->jctx.lock); - kbasep_remove_waiting_soft_job(katom); - kbase_finish_soft_job(katom); - if (kbase_jd_done_nolock(katom, true)) - kbase_js_sched_all(kctx->kbdev); - mutex_unlock(&kctx->jctx.lock); -} -#endif - -static void kbasep_soft_event_complete_job(struct work_struct *work) -{ - struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, work); - struct kbase_context *kctx = katom->kctx; - int resched; - - mutex_lock(&kctx->jctx.lock); - resched = kbase_jd_done_nolock(katom, true); - mutex_unlock(&kctx->jctx.lock); - - if (resched) - kbase_js_sched_all(kctx->kbdev); -} - -void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt) -{ - int cancel_timer = 1; - struct list_head *entry, *tmp; - unsigned long lflags; - - spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { - struct kbase_jd_atom *katom = list_entry(entry, struct kbase_jd_atom, queue); - - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_EVENT_WAIT: - if (katom->jc == evt) { - list_del(&katom->queue); - - katom->event_code = BASE_JD_EVENT_DONE; - INIT_WORK(&katom->work, kbasep_soft_event_complete_job); - queue_work(kctx->jctx.job_done_wq, &katom->work); - } else { - /* There are still other waiting jobs, we cannot - * cancel the timer yet. - */ - cancel_timer = 0; - } - break; -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG - case BASE_JD_REQ_SOFT_FENCE_WAIT: - /* Keep the timer running if fence debug is enabled and - * there are waiting fence jobs. - */ - cancel_timer = 0; - break; -#endif - } - } - - if (cancel_timer) - del_timer(&kctx->soft_job_timeout); - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -} - -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG -static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct device *dev = kctx->kbdev->dev; - int i; - - for (i = 0; i < 2; i++) { - struct kbase_jd_atom *dep; - - list_for_each_entry(dep, &katom->dep_head[i], dep_item[i]) { - if (dep->status == KBASE_JD_ATOM_STATE_UNUSED || - dep->status == KBASE_JD_ATOM_STATE_COMPLETED) - continue; - - if ((dep->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == - BASE_JD_REQ_SOFT_FENCE_TRIGGER) { - /* Found blocked trigger fence. */ - struct kbase_sync_fence_info info; - - if (!kbase_sync_fence_in_info_get(dep, &info)) { - dev_warn(dev, - "\tVictim trigger atom %d fence [%pK] %s: %s\n", - kbase_jd_atom_id(kctx, dep), info.fence, info.name, - kbase_sync_status_string(info.status)); - } - } - - kbase_fence_debug_check_atom(dep); - } - } -} - -static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct device *dev = katom->kctx->kbdev->dev; - int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); - unsigned long lflags; - struct kbase_sync_fence_info info; - - spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - - if (kbase_sync_fence_in_info_get(katom, &info)) { - /* Fence must have signaled just after timeout. */ - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); - return; - } - - dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%pK] after %dms\n", kctx->tgid, - kctx->id, kbase_jd_atom_id(kctx, katom), info.fence, timeout_ms); - dev_warn(dev, "\tGuilty fence [%pK] %s: %s\n", info.fence, info.name, - kbase_sync_status_string(info.status)); - - /* Search for blocked trigger atoms */ - kbase_fence_debug_check_atom(katom); - - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); - - kbase_sync_fence_in_dump(katom); -} - -struct kbase_fence_debug_work { - struct kbase_jd_atom *katom; - struct work_struct work; -}; - -static void kbase_fence_debug_wait_timeout_worker(struct work_struct *work) -{ - struct kbase_fence_debug_work *w = container_of(work, struct kbase_fence_debug_work, work); - struct kbase_jd_atom *katom = w->katom; - struct kbase_context *kctx = katom->kctx; - - mutex_lock(&kctx->jctx.lock); - kbase_fence_debug_wait_timeout(katom); - mutex_unlock(&kctx->jctx.lock); - - kfree(w); -} - -static void kbase_fence_debug_timeout(struct kbase_jd_atom *katom) -{ - struct kbase_fence_debug_work *work; - struct kbase_context *kctx = katom->kctx; - - /* Enqueue fence debug worker. Use job_done_wq to get - * debug print ordered with job completion. - */ - work = kzalloc(sizeof(struct kbase_fence_debug_work), GFP_ATOMIC); - /* Ignore allocation failure. */ - if (work) { - work->katom = katom; - INIT_WORK(&work->work, kbase_fence_debug_wait_timeout_worker); - queue_work(kctx->jctx.job_done_wq, &work->work); - } -} -#endif /* CONFIG_MALI_VALHALL_FENCE_DEBUG */ - -void kbasep_soft_job_timeout_worker(struct timer_list *timer) -{ - struct kbase_context *kctx = container_of(timer, struct kbase_context, soft_job_timeout); - u32 timeout_ms = (u32)atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); - ktime_t cur_time = ktime_get_raw(); - bool restarting = false; - unsigned long lflags; - struct list_head *entry, *tmp; - - spin_lock_irqsave(&kctx->waiting_soft_jobs_lock, lflags); - list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { - struct kbase_jd_atom *katom = list_entry(entry, struct kbase_jd_atom, queue); - s64 elapsed_time = ktime_to_ms(ktime_sub(cur_time, katom->start_timestamp)); - - if (elapsed_time < (s64)timeout_ms) { - restarting = true; - continue; - } - - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_EVENT_WAIT: - /* Take it out of the list to ensure that it - * will be cancelled in all cases - */ - list_del(&katom->queue); - - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - INIT_WORK(&katom->work, kbasep_soft_event_complete_job); - queue_work(kctx->jctx.job_done_wq, &katom->work); - break; -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG - case BASE_JD_REQ_SOFT_FENCE_WAIT: - kbase_fence_debug_timeout(katom); - break; -#endif - } - } - - if (restarting) - mod_timer(timer, jiffies + msecs_to_jiffies(timeout_ms)); - spin_unlock_irqrestore(&kctx->waiting_soft_jobs_lock, lflags); -} - -static int kbasep_soft_event_wait(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - unsigned char status; - - /* The status of this soft-job is stored in jc */ - if (kbasep_read_soft_event_status(kctx, katom->jc, &status)) { - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - return 0; - } - - if (status == BASE_JD_SOFT_EVENT_SET) - return 0; /* Event already set, nothing to do */ - - kbasep_add_waiting_with_timeout(katom); - - return 1; -} - -static void kbasep_soft_event_update_locked(struct kbase_jd_atom *katom, unsigned char new_status) -{ - /* Complete jobs waiting on the same event */ - struct kbase_context *kctx = katom->kctx; - - if (kbasep_write_soft_event_status(kctx, katom->jc, new_status) != 0) { - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - return; - } - - if (new_status == BASE_JD_SOFT_EVENT_SET) - kbasep_complete_triggered_soft_events(kctx, katom->jc); -} - -/** - * kbase_soft_event_update() - Update soft event state - * @kctx: Pointer to context - * @event: Event to update - * @new_status: New status value of event - * - * Update the event, and wake up any atoms waiting for the event. - * - * Return: 0 on success, a negative error code on failure. - */ -int kbase_soft_event_update(struct kbase_context *kctx, u64 event, unsigned char new_status) -{ - int err = 0; - - mutex_lock(&kctx->jctx.lock); - - if (kbasep_write_soft_event_status(kctx, event, new_status)) { - err = -ENOENT; - goto out; - } - - if (new_status == BASE_JD_SOFT_EVENT_SET) - kbasep_complete_triggered_soft_events(kctx, event); - -out: - mutex_unlock(&kctx->jctx.lock); - - return err; -} - -static void kbasep_soft_event_cancel_job(struct kbase_jd_atom *katom) -{ - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - if (kbase_jd_done_nolock(katom, true)) - kbase_js_sched_all(katom->kctx->kbdev); -} - -#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST -static void kbase_debug_copy_finish(struct kbase_jd_atom *katom) -{ - struct kbase_debug_copy_buffer *buffers = katom->softjob_data; - unsigned int i; - unsigned int nr = katom->nr_extres; - - if (!buffers) - return; - - kbase_gpu_vm_lock(katom->kctx); - for (i = 0; i < nr; i++) { - uint p; - struct kbase_mem_phy_alloc *gpu_alloc = buffers[i].gpu_alloc; - - if (!buffers[i].pages) - break; - for (p = 0; p < buffers[i].nr_pages; p++) { - struct page *pg = buffers[i].pages[p]; - - if (pg) - put_page(pg); - } - if (buffers[i].is_vmalloc) - vfree(buffers[i].pages); - else - kfree(buffers[i].pages); - if (gpu_alloc) { - switch (gpu_alloc->type) { - case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { - kbase_free_user_buffer(&buffers[i]); - break; - } - default: - /* Nothing to be done. */ - break; - } - kbase_mem_phy_alloc_put(gpu_alloc); - } - } - kbase_gpu_vm_unlock(katom->kctx); - kfree(buffers); - - katom->softjob_data = NULL; -} - -static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) -{ - struct kbase_debug_copy_buffer *buffers; - struct base_jd_debug_copy_buffer *user_buffers = NULL; - unsigned int i; - unsigned int nr = katom->nr_extres; - int ret = 0; - void __user *user_structs = (void __user *)(uintptr_t)katom->jc; - size_t copy_size; - - if (!user_structs) - return -EINVAL; - - buffers = kcalloc(nr, sizeof(*buffers), GFP_KERNEL); - if (!buffers) { - ret = -ENOMEM; - goto out_cleanup; - } - katom->softjob_data = buffers; - - user_buffers = kmalloc_array(nr, sizeof(*user_buffers), GFP_KERNEL); - - if (!user_buffers) { - ret = -ENOMEM; - goto out_cleanup; - } - - if (check_mul_overflow(sizeof(*user_buffers), (size_t)nr, ©_size)) { - ret = -EINVAL; - goto out_cleanup; - } - - ret = copy_from_user(user_buffers, user_structs, copy_size); - if (ret) { - ret = -EFAULT; - goto out_cleanup; - } - - for (i = 0; i < nr; i++) { - u64 addr = user_buffers[i].address; - u64 page_addr = addr & PAGE_MASK; - u64 end_page_addr = addr + user_buffers[i].size - 1; - u64 last_page_addr = end_page_addr & PAGE_MASK; - size_t nr_pages = (last_page_addr - page_addr) / PAGE_SIZE + 1; - int pinned_pages; - struct kbase_va_region *reg; - struct base_external_resource user_extres; - - if (!addr) - continue; - - if (last_page_addr < page_addr) { - ret = -EINVAL; - goto out_cleanup; - } - - buffers[i].nr_pages = nr_pages; - buffers[i].offset = addr & ~PAGE_MASK; - if (buffers[i].offset >= PAGE_SIZE) { - ret = -EINVAL; - goto out_cleanup; - } - buffers[i].size = user_buffers[i].size; - - if (nr_pages > (KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD / sizeof(struct page *))) { - buffers[i].is_vmalloc = true; - buffers[i].pages = vzalloc(nr_pages * sizeof(struct page *)); - } else { - buffers[i].is_vmalloc = false; - buffers[i].pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); - } - - if (!buffers[i].pages) { - ret = -ENOMEM; - goto out_cleanup; - } - - pinned_pages = get_user_pages_fast(page_addr, nr_pages, 1, /* Write */ - buffers[i].pages); - if (pinned_pages < 0) { - /* get_user_pages_fast has failed - page array is not - * valid. Don't try to release any pages. - */ - buffers[i].nr_pages = 0; - - ret = pinned_pages; - goto out_cleanup; - } - if ((size_t)pinned_pages != nr_pages) { - /* Adjust number of pages, so that we only attempt to - * release pages in the array that we know are valid. - */ - buffers[i].nr_pages = (unsigned int)pinned_pages; - - ret = -EINVAL; - goto out_cleanup; - } - - user_extres = user_buffers[i].extres; - if (user_extres.ext_resource == 0ULL) { - ret = -EINVAL; - goto out_cleanup; - } - - kbase_gpu_vm_lock(katom->kctx); - reg = kbase_region_tracker_find_region_enclosing_address( - katom->kctx, - user_extres.ext_resource & ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE); - - if (kbase_is_region_invalid_or_free(reg) || reg->gpu_alloc == NULL) { - ret = -EINVAL; - goto out_unlock; - } - - buffers[i].gpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); - buffers[i].nr_extres_pages = reg->nr_pages; - - if (reg->nr_pages * PAGE_SIZE != buffers[i].size) - dev_warn( - katom->kctx->kbdev->dev, - "Copy buffer is not of same size as the external resource to copy.\n"); - - switch (reg->gpu_alloc->type) { - case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { - struct kbase_mem_phy_alloc *alloc = reg->gpu_alloc; - const unsigned long nr_pages = alloc->imported.user_buf.nr_pages; - const unsigned long start = alloc->imported.user_buf.address; - - if (alloc->imported.user_buf.mm != current->mm) { - ret = -EINVAL; - goto out_unlock; - } - buffers[i].extres_pages = - kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); - if (!buffers[i].extres_pages) { - ret = -ENOMEM; - goto out_unlock; - } - kbase_gpu_vm_unlock(katom->kctx); - ret = get_user_pages_fast(start, nr_pages, 0, buffers[i].extres_pages); - kbase_gpu_vm_lock(katom->kctx); - if (ret != (int)nr_pages) { - /* Adjust number of pages, so that we only - * attempt to release pages in the array that we - * know are valid. - */ - if (ret < 0) - buffers[i].nr_extres_pages = 0; - else - buffers[i].nr_extres_pages = (unsigned int)ret; - - goto out_unlock; - } - ret = 0; - break; - } - default: - /* Nothing to be done. */ - break; - } - kbase_gpu_vm_unlock(katom->kctx); - } - kfree(user_buffers); - - return ret; - -out_unlock: - kbase_gpu_vm_unlock(katom->kctx); - -out_cleanup: - /* Frees allocated memory for kbase_debug_copy_job struct, including - * members, and sets jc to 0 - */ - kbase_debug_copy_finish(katom); - kfree(user_buffers); - - return ret; -} - -#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE -static void *dma_buf_kmap_page(struct kbase_mem_phy_alloc *gpu_alloc, unsigned long page_num, - struct page **page) -{ - struct sg_table *sgt = gpu_alloc->imported.umm.sgt; - struct sg_page_iter sg_iter; - unsigned long page_index = 0; - - if (WARN_ON(gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) - return NULL; - - if (!sgt) - return NULL; - - if (WARN_ON(page_num >= gpu_alloc->nents)) - return NULL; - - for_each_sg_page(sgt->sgl, &sg_iter, sgt->nents, 0) { - if (page_index == page_num) { - *page = sg_page_iter_page(&sg_iter); - - return kbase_kmap(*page); - } - page_index++; - } - - return NULL; -} -#endif - -/** - * kbase_mem_copy_from_extres() - Copy from external resources. - * - * @kctx: kbase context within which the copying is to take place. - * @buf_data: Pointer to the information about external resources: - * pages pertaining to the external resource, number of - * pages to copy. - * - * Return: 0 on success, error code otherwise. - */ -static int kbase_mem_copy_from_extres(struct kbase_context *kctx, - struct kbase_debug_copy_buffer *buf_data) -{ - unsigned int i; - unsigned int target_page_nr = 0; - struct page **pages = buf_data->pages; - u64 offset = buf_data->offset; - size_t extres_size = buf_data->nr_extres_pages * PAGE_SIZE; - size_t to_copy = min(extres_size, buf_data->size); - struct kbase_mem_phy_alloc *gpu_alloc = buf_data->gpu_alloc; - int ret = 0; - size_t dma_to_copy; - - KBASE_DEBUG_ASSERT(pages != NULL); - - kbase_gpu_vm_lock(kctx); - if (!gpu_alloc) { - ret = -EINVAL; - goto out_unlock; - } - - switch (gpu_alloc->type) { - case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { - for (i = 0; i < buf_data->nr_extres_pages && target_page_nr < buf_data->nr_pages; - i++) { - struct page *pg = buf_data->extres_pages[i]; - void *extres_page = kbase_kmap(pg); - if (extres_page) { - ret = kbase_mem_copy_to_pinned_user_pages(pages, extres_page, - &to_copy, - buf_data->nr_pages, - &target_page_nr, offset); - kbase_kunmap(pg, extres_page); - if (ret) - goto out_unlock; - } - } - } break; - case KBASE_MEM_TYPE_IMPORTED_UMM: { - struct dma_buf *dma_buf = gpu_alloc->imported.umm.dma_buf; - - KBASE_DEBUG_ASSERT(dma_buf != NULL); - if (dma_buf->size > buf_data->nr_extres_pages * PAGE_SIZE) - dev_warn(kctx->kbdev->dev, "External resources buffer size mismatch"); - - dma_to_copy = min(dma_buf->size, (size_t)(buf_data->nr_extres_pages * PAGE_SIZE)); - ret = dma_buf_begin_cpu_access(dma_buf, DMA_FROM_DEVICE); - if (ret) - goto out_unlock; - - for (i = 0; i < dma_to_copy / PAGE_SIZE && target_page_nr < buf_data->nr_pages; - i++) { -#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE - struct page *pg; - void *extres_page = dma_buf_kmap_page(gpu_alloc, i, &pg); -#else - void *extres_page = dma_buf_kmap(dma_buf, i); -#endif - if (extres_page) { - ret = kbase_mem_copy_to_pinned_user_pages(pages, extres_page, - &to_copy, - buf_data->nr_pages, - &target_page_nr, offset); - -#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE - kbase_kunmap(pg, extres_page); -#else - dma_buf_kunmap(dma_buf, i, extres_page); -#endif - if (ret) - break; - } - } - dma_buf_end_cpu_access(dma_buf, DMA_FROM_DEVICE); - break; - } - default: - ret = -EINVAL; - } -out_unlock: - kbase_gpu_vm_unlock(kctx); - return ret; -} - -static int kbase_debug_copy(struct kbase_jd_atom *katom) -{ - struct kbase_debug_copy_buffer *buffers = katom->softjob_data; - unsigned int i; - - if (WARN_ON(!buffers)) - return -EINVAL; - - for (i = 0; i < katom->nr_extres; i++) { - int res = kbase_mem_copy_from_extres(katom->kctx, &buffers[i]); - - if (res) - return res; - } - - return 0; -} -#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */ -#endif /* !MALI_USE_CSF */ - #define KBASEP_JIT_ALLOC_GPU_ADDR_ALIGNMENT ((u32)0x7) int kbasep_jit_alloc_validate(struct kbase_context *kctx, struct base_jit_alloc_info *info) @@ -910,817 +85,5 @@ int kbasep_jit_alloc_validate(struct kbase_context *kctx, struct base_jit_alloc_ return -EINVAL; #endif -#if !MALI_USE_CSF - /* If BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE is set, heap_info_gpu_addr - * cannot be 0 - */ - if ((info->flags & BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) && !info->heap_info_gpu_addr) - return -EINVAL; -#endif /* !MALI_USE_CSF */ - return 0; } - -#if !MALI_USE_CSF - -static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) -{ - __user u8 *data = (__user u8 *)(uintptr_t)katom->jc; - struct base_jit_alloc_info *info; - struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - u32 count; - int ret; - u32 i; - - if (!kbase_mem_allow_alloc(kctx)) { - dev_dbg(kbdev->dev, "Invalid attempt to allocate JIT memory by %s/%d for ctx %d_%d", - current->comm, current->pid, kctx->tgid, kctx->id); - ret = -EINVAL; - goto fail; - } - - /* For backwards compatibility, and to prevent reading more than 1 jit - * info struct on jit version 1 - */ - if (katom->nr_extres == 0) - katom->nr_extres = 1; - count = katom->nr_extres; - - /* Sanity checks */ - if (!data || count > kctx->jit_max_allocations || count > ARRAY_SIZE(kctx->jit_alloc)) { - ret = -EINVAL; - goto fail; - } - - /* Copy the information for safe access and future storage */ - info = kmalloc_array(count, sizeof(*info), GFP_KERNEL); - if (!info) { - ret = -ENOMEM; - goto fail; - } - - katom->softjob_data = info; - - for (i = 0; i < count; i++, info++, data += sizeof(*info)) { - if (copy_from_user(info, data, sizeof(*info)) != 0) { - ret = -EINVAL; - goto free_info; - } - - ret = kbasep_jit_alloc_validate(kctx, info); - if (ret) - goto free_info; - KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO( - kbdev, katom, info->va_pages, info->commit_pages, info->extension, info->id, - info->bin_id, info->max_allocations, info->flags, info->usage_id); - } - - katom->jit_blocked = false; - - lockdep_assert_held(&kctx->jctx.lock); - list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head); - - /* - * Note: - * The provided info->gpu_alloc_addr isn't validated here as - * userland can cache allocations which means that even - * though the region is valid it doesn't represent the - * same thing it used to. - * - * Complete validation of va_pages, commit_pages and extension - * isn't done here as it will be done during the call to - * kbase_mem_alloc. - */ - return 0; - -free_info: - kfree(katom->softjob_data); - katom->softjob_data = NULL; -fail: - return ret; -} - -static u8 *kbase_jit_free_get_ids(struct kbase_jd_atom *katom) -{ - if (WARN_ON((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) != BASE_JD_REQ_SOFT_JIT_FREE)) - return NULL; - - return (u8 *)katom->softjob_data; -} - -static void kbase_jit_add_to_pending_alloc_list(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct list_head *target_list_head = NULL; - struct kbase_jd_atom *entry; - - list_for_each_entry(entry, &kctx->jctx.jit_pending_alloc, queue) { - if (katom->age < entry->age) { - target_list_head = &entry->queue; - break; - } - } - - if (target_list_head == NULL) - target_list_head = &kctx->jctx.jit_pending_alloc; - - list_add_tail(&katom->queue, target_list_head); -} - -static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - struct base_jit_alloc_info *info; - struct kbase_va_region *reg; - struct kbase_vmap_struct mapping; - u64 *ptr, new_addr; - u32 count = katom->nr_extres; - u32 i; - bool ignore_pressure_limit = false; - - trace_sysgraph(SGR_SUBMIT, kctx->id, kbase_jd_atom_id(kctx, katom)); - - if (katom->jit_blocked) { - list_del(&katom->queue); - katom->jit_blocked = false; - } - - info = katom->softjob_data; - if (WARN_ON(!info)) { - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return 0; - } - - for (i = 0; i < count; i++, info++) { - /* The JIT ID is still in use so fail the allocation */ - if (kctx->jit_alloc[info->id]) { - katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; - return 0; - } - } - -#if MALI_JIT_PRESSURE_LIMIT_BASE - /* - * If this is the only JIT_ALLOC atom in-flight or if JIT pressure limit - * is disabled at the context scope, then bypass JIT pressure limit - * logic in kbase_jit_allocate(). - */ - if (!kbase_ctx_flag(kctx, KCTX_JPL_ENABLED) || (kctx->jit_current_allocations == 0)) { - ignore_pressure_limit = true; - } -#else - ignore_pressure_limit = true; -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - for (i = 0, info = katom->softjob_data; i < count; i++, info++) { - if (kctx->jit_alloc[info->id]) { - /* The JIT ID is duplicated in this atom. Roll back - * previous allocations and fail. - */ - u32 j; - - info = katom->softjob_data; - for (j = 0; j < i; j++, info++) { - kbase_jit_free(kctx, kctx->jit_alloc[info->id]); - kctx->jit_alloc[info->id] = KBASE_RESERVED_REG_JIT_ALLOC; - } - - katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; - return 0; - } - - /* Create a JIT allocation */ - reg = kbase_jit_allocate(kctx, info, ignore_pressure_limit); - if (!reg) { - struct kbase_jd_atom *jit_atom; - bool can_block = false; - - lockdep_assert_held(&kctx->jctx.lock); - - list_for_each_entry(jit_atom, &kctx->jctx.jit_atoms_head, jit_node) { - if (jit_atom == katom) - break; - - if ((jit_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == - BASE_JD_REQ_SOFT_JIT_FREE) { - u8 *free_ids = kbase_jit_free_get_ids(jit_atom); - - if (free_ids && *free_ids && kctx->jit_alloc[*free_ids]) { - /* A JIT free which is active and - * submitted before this atom - */ - can_block = true; - break; - } - } - } - - if (!can_block) { - /* Mark the failed allocation as well as the - * other un-attempted allocations in the set, - * so we know they are in use even if the - * allocation itself failed. - */ - for (; i < count; i++, info++) { - kctx->jit_alloc[info->id] = KBASE_RESERVED_REG_JIT_ALLOC; - } - - katom->event_code = BASE_JD_EVENT_MEM_GROWTH_FAILED; - dev_warn_ratelimited(kbdev->dev, - "JIT alloc softjob failed: atom id %d\n", - kbase_jd_atom_id(kctx, katom)); - return 0; - } - - /* There are pending frees for an active allocation - * so we should wait to see whether they free the - * memory. Add to the list of atoms for which JIT - * allocation is pending. - */ - kbase_jit_add_to_pending_alloc_list(katom); - katom->jit_blocked = true; - - /* Rollback, the whole set will be re-attempted */ - while (i-- > 0) { - info--; - kbase_jit_free(kctx, kctx->jit_alloc[info->id]); - kctx->jit_alloc[info->id] = NULL; - } - - return 1; - } - - /* Bind it to the user provided ID. */ - kctx->jit_alloc[info->id] = reg; - } - - for (i = 0, info = katom->softjob_data; i < count; i++, info++) { - u64 entry_mmu_flags = 0; - /* - * Write the address of the JIT allocation to the user provided - * GPU allocation. - */ - ptr = kbase_vmap_prot(kctx, info->gpu_alloc_addr, sizeof(*ptr), KBASE_REG_CPU_WR, - &mapping); - if (!ptr) { - /* - * Leave the allocations "live" as the JIT free atom - * will be submitted anyway. - */ - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return 0; - } - - reg = kctx->jit_alloc[info->id]; - new_addr = reg->start_pfn << PAGE_SHIFT; - *ptr = new_addr; - -#if defined(CONFIG_MALI_VECTOR_DUMP) - /* - * Retrieve the mmu flags for JIT allocation - * only if dumping is enabled - */ - entry_mmu_flags = kbase_mmu_create_ate(kbdev, (struct tagged_addr){ 0 }, reg->flags, - MIDGARD_MMU_BOTTOMLEVEL, kctx->jit_group_id); -#endif - - KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(kbdev, katom, info->gpu_alloc_addr, new_addr, - info->flags, entry_mmu_flags, info->id, - info->commit_pages, info->extension, - info->va_pages); - kbase_vunmap(kctx, &mapping); - - kbase_trace_jit_report_gpu_mem(kctx, reg, KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); - } - - katom->event_code = BASE_JD_EVENT_DONE; - - return 0; -} - -static void kbase_jit_allocate_finish(struct kbase_jd_atom *katom) -{ - struct base_jit_alloc_info *info; - - lockdep_assert_held(&katom->kctx->jctx.lock); - - if (WARN_ON(!katom->softjob_data)) - return; - - /* Remove atom from jit_atoms_head list */ - list_del(&katom->jit_node); - - if (katom->jit_blocked) { - list_del(&katom->queue); - katom->jit_blocked = false; - } - - info = katom->softjob_data; - /* Free the info structure */ - kfree(info); -} - -static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - __user void *data = (__user void *)(uintptr_t)katom->jc; - u8 *ids; - u32 count = MAX(katom->nr_extres, 1); - u32 i; - int ret; - size_t copy_size; - - /* Sanity checks */ - if (count > ARRAY_SIZE(kctx->jit_alloc)) { - ret = -EINVAL; - goto fail; - } - - /* Copy the information for safe access and future storage */ - ids = kmalloc_array(count, sizeof(*ids), GFP_KERNEL); - if (!ids) { - ret = -ENOMEM; - goto fail; - } - - lockdep_assert_held(&kctx->jctx.lock); - katom->softjob_data = ids; - - /* For backwards compatibility */ - if (katom->nr_extres) { - /* Fail the job if there is no list of ids */ - if (!data) { - ret = -EINVAL; - goto free_info; - } - if (check_mul_overflow(sizeof(*ids), (size_t)count, ©_size)) { - ret = -EINVAL; - goto free_info; - } - - if (copy_from_user(ids, data, copy_size) != 0) { - ret = -EINVAL; - goto free_info; - } - } else { - katom->nr_extres = 1; - *ids = (u8)katom->jc; - } - for (i = 0; i < count; i++) - KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITFREEINFO(kbdev, katom, ids[i]); - - list_add_tail(&katom->jit_node, &kctx->jctx.jit_atoms_head); - - return 0; - -free_info: - kfree(katom->softjob_data); - katom->softjob_data = NULL; -fail: - return ret; -} - -static void kbase_jit_free_process(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - u8 *ids = kbase_jit_free_get_ids(katom); - u32 count = katom->nr_extres; - u32 i; - - if (ids == NULL) { - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return; - } - - for (i = 0; i < count; i++, ids++) { - /* - * If the ID is zero or it is not in use yet then fail the job. - */ - if ((*ids == 0) || (kctx->jit_alloc[*ids] == NULL)) { - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return; - } - } -} - -static void kbasep_jit_finish_worker(struct work_struct *work) -{ - struct kbase_jd_atom *katom = container_of(work, struct kbase_jd_atom, work); - struct kbase_context *kctx = katom->kctx; - int resched; - - mutex_lock(&kctx->jctx.lock); - kbase_finish_soft_job(katom); - resched = kbase_jd_done_nolock(katom, true); - mutex_unlock(&kctx->jctx.lock); - - if (resched) - kbase_js_sched_all(kctx->kbdev); -} - -void kbase_jit_retry_pending_alloc(struct kbase_context *kctx) -{ - LIST_HEAD(jit_pending_alloc_list); - struct list_head *i, *tmp; - - list_splice_tail_init(&kctx->jctx.jit_pending_alloc, &jit_pending_alloc_list); - - list_for_each_safe(i, tmp, &jit_pending_alloc_list) { - struct kbase_jd_atom *pending_atom = list_entry(i, struct kbase_jd_atom, queue); - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kctx->kbdev, pending_atom); - kbase_kinstr_jm_atom_sw_start(pending_atom); - if (kbase_jit_allocate_process(pending_atom) == 0) { - /* Atom has completed */ - INIT_WORK(&pending_atom->work, kbasep_jit_finish_worker); - queue_work(kctx->jctx.job_done_wq, &pending_atom->work); - } - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END(kctx->kbdev, pending_atom); - kbase_kinstr_jm_atom_sw_stop(pending_atom); - } -} - -static void kbase_jit_free_finish(struct kbase_jd_atom *katom) -{ - struct kbase_context *kctx = katom->kctx; - u8 *ids; - size_t j; - - lockdep_assert_held(&kctx->jctx.lock); - - ids = kbase_jit_free_get_ids(katom); - if (WARN_ON(ids == NULL)) - return; - - /* Remove this atom from the jit_atoms_head list */ - list_del(&katom->jit_node); - - for (j = 0; j != katom->nr_extres; ++j) { - if ((ids[j] != 0) && (kctx->jit_alloc[ids[j]] != NULL)) { - /* - * If the ID is valid but the allocation request failed - * still succeed this soft job but don't try and free - * the allocation. - */ - if (kctx->jit_alloc[ids[j]] != KBASE_RESERVED_REG_JIT_ALLOC) { - KBASE_TLSTREAM_TL_JIT_USEDPAGES( - kctx->kbdev, kctx->jit_alloc[ids[j]]->gpu_alloc->nents, - ids[j]); - kbase_jit_free(kctx, kctx->jit_alloc[ids[j]]); - } - kctx->jit_alloc[ids[j]] = NULL; - } - } - /* Free the list of ids */ - kfree(ids); - - kbase_jit_retry_pending_alloc(kctx); -} - -static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) -{ - __user struct base_external_resource_list *user_ext_res; - struct kbase_external_resource_list *ext_res; - u64 count = 0; - size_t copy_size; - - user_ext_res = (__user struct base_external_resource_list *)(uintptr_t)katom->jc; - - /* Fail the job if there is no info structure */ - if (!user_ext_res) - return -EINVAL; - - if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) - return -EINVAL; - - /* Is the number of external resources in range? */ - if (!count || count > BASE_EXT_RES_COUNT_MAX) - return -EINVAL; - - /* Copy the information for safe access and future storage */ - copy_size = sizeof(struct base_external_resource) * count; - ext_res = kmalloc(sizeof(*ext_res) + copy_size, GFP_KERNEL); - if (!ext_res) - return -ENOMEM; - - ext_res->count = count; - ext_res->ext_res = (struct base_external_resource *)(ext_res + 1); - if (copy_from_user(ext_res->ext_res, user_ext_res->ext_res, copy_size) != 0) { - kfree(ext_res); - return -EINVAL; - } - - katom->softjob_data = ext_res; - - return 0; -} - -static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) -{ - struct kbase_external_resource_list *ext_res; - uint64_t i; - bool failed = false; - - ext_res = katom->softjob_data; - if (!ext_res) - goto failed_jc; - - kbase_gpu_vm_lock(katom->kctx); - - for (i = 0; i < ext_res->count; i++) { - u64 gpu_addr; - - gpu_addr = ext_res->ext_res[i].ext_resource & ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE; - if (map) { - if (!kbase_sticky_resource_acquire(katom->kctx, gpu_addr, NULL)) - goto failed_loop; - } else { - if (!kbase_sticky_resource_release_force(katom->kctx, NULL, gpu_addr)) - failed = true; - } - } - - /* - * In the case of unmap we continue unmapping other resources in the - * case of failure but will always report failure if _any_ unmap - * request fails. - */ - if (failed) - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - else - katom->event_code = BASE_JD_EVENT_DONE; - - kbase_gpu_vm_unlock(katom->kctx); - - return; - -failed_loop: - while (i > 0) { - u64 const gpu_addr = ext_res->ext_res[i - 1].ext_resource & - ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE; - - kbase_sticky_resource_release_force(katom->kctx, NULL, gpu_addr); - - --i; - } - - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - kbase_gpu_vm_unlock(katom->kctx); - -failed_jc: - return; -} - -static void kbase_ext_res_finish(struct kbase_jd_atom *katom) -{ - struct kbase_external_resource_list *ext_res; - - ext_res = katom->softjob_data; - /* Free the info structure */ - kfree(ext_res); -} - -int kbase_process_soft_job(struct kbase_jd_atom *katom) -{ - int ret = 0; - struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kbdev, katom); - kbase_kinstr_jm_atom_sw_start(katom); - - trace_sysgraph(SGR_SUBMIT, kctx->id, kbase_jd_atom_id(kctx, katom)); - - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: - ret = kbase_dump_cpu_gpu_time(katom); - break; - -#if IS_ENABLED(CONFIG_SYNC_FILE) - case BASE_JD_REQ_SOFT_FENCE_TRIGGER: - katom->event_code = kbase_sync_fence_out_trigger( - katom, katom->event_code == BASE_JD_EVENT_DONE ? 0 : -EFAULT); - break; - case BASE_JD_REQ_SOFT_FENCE_WAIT: { - ret = kbase_sync_fence_in_wait(katom); - - if (ret == 1) { -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG - kbasep_add_waiting_with_timeout(katom); -#else - kbasep_add_waiting_soft_job(katom); -#endif - } - break; - } -#endif - case BASE_JD_REQ_SOFT_EVENT_WAIT: - ret = kbasep_soft_event_wait(katom); - break; - case BASE_JD_REQ_SOFT_EVENT_SET: - kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_SET); - break; - case BASE_JD_REQ_SOFT_EVENT_RESET: - kbasep_soft_event_update_locked(katom, BASE_JD_SOFT_EVENT_RESET); - break; -#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST - case BASE_JD_REQ_SOFT_DEBUG_COPY: { - int res = kbase_debug_copy(katom); - - if (res) - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - break; - } -#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */ - case BASE_JD_REQ_SOFT_JIT_ALLOC: - ret = kbase_jit_allocate_process(katom); - break; - case BASE_JD_REQ_SOFT_JIT_FREE: - kbase_jit_free_process(katom); - break; - case BASE_JD_REQ_SOFT_EXT_RES_MAP: - kbase_ext_res_process(katom, true); - break; - case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: - kbase_ext_res_process(katom, false); - break; - } - - /* Atom is complete */ - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END(kbdev, katom); - kbase_kinstr_jm_atom_sw_stop(katom); - return ret; -} - -void kbase_cancel_soft_job(struct kbase_jd_atom *katom) -{ - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { -#if IS_ENABLED(CONFIG_SYNC_FILE) - case BASE_JD_REQ_SOFT_FENCE_WAIT: - kbase_sync_fence_in_cancel_wait(katom); - break; -#endif - case BASE_JD_REQ_SOFT_EVENT_WAIT: - kbasep_soft_event_cancel_job(katom); - break; - default: - /* This soft-job doesn't support cancellation! */ - KBASE_DEBUG_ASSERT(0); - } -} - -int kbase_prepare_soft_job(struct kbase_jd_atom *katom) -{ - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: { - if (!IS_ALIGNED(katom->jc, cache_line_size())) - return -EINVAL; - } break; -#if IS_ENABLED(CONFIG_SYNC_FILE) - case BASE_JD_REQ_SOFT_FENCE_TRIGGER: { - struct base_fence fence; - int fd; - - if (copy_from_user(&fence, (__user void *)(uintptr_t)katom->jc, sizeof(fence)) != 0) - return -EINVAL; - - fd = kbase_sync_fence_out_create(katom, fence.basep.stream_fd); - if (fd < 0) - return -EINVAL; - - fence.basep.fd = fd; - if (copy_to_user((__user void *)(uintptr_t)katom->jc, &fence, sizeof(fence)) != 0) { - kbase_sync_fence_out_remove(katom); - /* fd should have been closed here, but there's - * no good way of doing that. Since - * copy_to_user() very rarely fails, and the fd - * will get closed on process termination this - * won't be a problem. - */ - fence.basep.fd = -EINVAL; - return -EINVAL; - } - } break; - case BASE_JD_REQ_SOFT_FENCE_WAIT: { - struct base_fence fence; - int ret; - - if (copy_from_user(&fence, (__user void *)(uintptr_t)katom->jc, sizeof(fence)) != 0) - return -EINVAL; - - /* Get a reference to the fence object */ - ret = kbase_sync_fence_in_from_fd(katom, fence.basep.fd); - if (ret < 0) - return ret; - } break; -#endif /* CONFIG_SYNC_FILE */ - case BASE_JD_REQ_SOFT_JIT_ALLOC: - return kbase_jit_allocate_prepare(katom); - case BASE_JD_REQ_SOFT_JIT_FREE: - return kbase_jit_free_prepare(katom); - case BASE_JD_REQ_SOFT_EVENT_WAIT: - case BASE_JD_REQ_SOFT_EVENT_SET: - case BASE_JD_REQ_SOFT_EVENT_RESET: - if (katom->jc == 0) - return -EINVAL; - break; -#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST - case BASE_JD_REQ_SOFT_DEBUG_COPY: - return kbase_debug_copy_prepare(katom); -#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */ - case BASE_JD_REQ_SOFT_EXT_RES_MAP: - return kbase_ext_res_prepare(katom); - case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: - return kbase_ext_res_prepare(katom); - default: - /* Unsupported soft-job */ - return -EINVAL; - } - return 0; -} - -void kbase_finish_soft_job(struct kbase_jd_atom *katom) -{ - trace_sysgraph(SGR_COMPLETE, katom->kctx->id, kbase_jd_atom_id(katom->kctx, katom)); - - switch (katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { - case BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME: - /* Nothing to do */ - break; -#if IS_ENABLED(CONFIG_SYNC_FILE) - case BASE_JD_REQ_SOFT_FENCE_TRIGGER: - /* If fence has not yet been signaled, do it now */ - kbase_sync_fence_out_trigger(katom, - katom->event_code == BASE_JD_EVENT_DONE ? 0 : -EFAULT); - break; - case BASE_JD_REQ_SOFT_FENCE_WAIT: - /* Release katom's reference to fence object */ - kbase_sync_fence_in_remove(katom); - break; -#endif /* CONFIG_SYNC_FILE */ -#if IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST - case BASE_JD_REQ_SOFT_DEBUG_COPY: - kbase_debug_copy_finish(katom); - break; -#endif /* IS_ENABLED(CONFIG_MALI_VECTOR_DUMP) || MALI_UNIT_TEST */ - case BASE_JD_REQ_SOFT_JIT_ALLOC: - kbase_jit_allocate_finish(katom); - break; - case BASE_JD_REQ_SOFT_EXT_RES_MAP: - kbase_ext_res_finish(katom); - break; - case BASE_JD_REQ_SOFT_EXT_RES_UNMAP: - kbase_ext_res_finish(katom); - break; - case BASE_JD_REQ_SOFT_JIT_FREE: - kbase_jit_free_finish(katom); - break; - } -} - -void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) -{ - LIST_HEAD(local_suspended_soft_jobs); - struct kbase_jd_atom *tmp_iter; - struct kbase_jd_atom *katom_iter; - struct kbasep_js_device_data *js_devdata; - bool resched = false; - - KBASE_DEBUG_ASSERT(kbdev); - - js_devdata = &kbdev->js_data; - - /* Move out the entire list */ - mutex_lock(&js_devdata->runpool_mutex); - list_splice_init(&js_devdata->suspended_soft_jobs_list, &local_suspended_soft_jobs); - mutex_unlock(&js_devdata->runpool_mutex); - - /* - * Each atom must be detached from the list and ran separately - - * it could be re-added to the old list, but this is unlikely - */ - list_for_each_entry_safe(katom_iter, tmp_iter, &local_suspended_soft_jobs, dep_item[1]) { - struct kbase_context *kctx = katom_iter->kctx; - - mutex_lock(&kctx->jctx.lock); - - /* Remove from the global list */ - list_del(&katom_iter->dep_item[1]); - /* Remove from the context's list of waiting soft jobs */ - kbasep_remove_waiting_soft_job(katom_iter); - - if (kbase_process_soft_job(katom_iter) == 0) { - kbase_finish_soft_job(katom_iter); - resched |= kbase_jd_done_nolock(katom_iter, true); - if (kbase_has_arbiter(kctx->kbdev)) - atomic_dec(&kbdev->pm.gpu_users_waiting); - } - mutex_unlock(&kctx->jctx.lock); - } - - if (resched) - kbase_js_sched_all(kbdev); -} -#endif /* !MALI_USE_CSF */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_sync.h b/drivers/gpu/arm/valhall/mali_kbase_sync.h index 350296c5da12..7a91a36471be 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_sync.h +++ b/drivers/gpu/arm/valhall/mali_kbase_sync.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -36,11 +36,6 @@ #include -#if !MALI_USE_CSF -struct kbase_jd_atom; -struct work_struct; -#endif - /** * struct kbase_sync_fence_info - Information about a fence * @fence: Pointer to fence (type is void*, as underlaying struct can differ) @@ -71,30 +66,6 @@ struct kbase_sync_fence_info { */ int kbase_sync_fence_stream_create(const char *name, int *const out_fd); -#if !MALI_USE_CSF -/** - * kbase_sync_fence_out_create - Create an explicit output fence to specified atom - * - * @katom: Atom to assign the new explicit fence to - * @stream_fd: File descriptor for stream object to create fence on - * - * Return: Valid file descriptor to fence or < 0 on error - */ -int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); - -/** - * kbase_sync_fence_in_from_fd() - Assigns an existing fence to specified atom - * @katom: Atom to assign the existing explicit fence to - * @fd: File descriptor to an existing fence - * - * Assigns an explicit input fence to atom. - * This can later be waited for by calling @kbase_sync_fence_in_wait - * - * Return: 0 on success, < 0 on error - */ -int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); -#endif /* !MALI_USE_CSF */ - /** * kbase_sync_fence_validate() - Validate a fd to be a valid fence * @@ -107,78 +78,6 @@ int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); */ int kbase_sync_fence_validate(int fd); -#if !MALI_USE_CSF -/** - * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom - * @katom: Atom with an explicit fence to signal - * @result: < 0 means signal with error, 0 >= indicates success - * - * Signal output fence attached on katom and remove the fence from the atom. - * - * Return: The "next" event code for atom, typically JOB_CANCELLED or EVENT_DONE - */ -enum base_jd_event_code kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result); - -/** - * kbase_sync_fence_in_wait() - Wait for explicit input fence to be signaled - * @katom: Atom with explicit fence to wait for - * - * If the fence is already signaled, then 0 is returned, and the caller must - * continue processing of the katom. - * - * If the fence isn't already signaled, then this kbase_sync framework will - * take responsibility to continue the processing once the fence is signaled. - * - * Return: 0 if already signaled, otherwise 1 - */ -int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom); - -/** - * kbase_sync_fence_in_cancel_wait() - Cancel explicit input fence waits - * @katom: Atom to cancel wait for - * - * This function is fully responsible for continuing processing of this atom - * (remove_waiting_soft_job + finish_soft_job + jd_done + js_sched_all) - */ -void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom); - -/** - * kbase_sync_fence_in_remove() - Remove the input fence from the katom - * @katom: Atom to remove explicit input fence for - * - * This will also release the corresponding reference. - */ -void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); - -/** - * kbase_sync_fence_out_remove() - Remove the output fence from the katom - * @katom: Atom to remove explicit output fence for - * - * This will also release the corresponding reference. - */ -void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); -#endif /* !MALI_USE_CSF */ - -#if !MALI_USE_CSF -/** - * kbase_sync_fence_in_info_get() - Retrieves information about input fence - * @katom: Atom to get fence information from - * @info: Struct to be filled with fence information - * - * Return: 0 on success, < 0 on error - */ -int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info); - -/** - * kbase_sync_fence_out_info_get() - Retrieves information about output fence - * @katom: Atom to get fence information from - * @info: Struct to be filled with fence information - * - * Return: 0 on success, < 0 on error - */ -int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info); -#endif /* !MALI_USE_CSF */ - #if IS_ENABLED(CONFIG_SYNC_FILE) void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_info *info); #endif @@ -191,20 +90,4 @@ void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_ */ const char *kbase_sync_status_string(int status); -#if !MALI_USE_CSF -/* - * Internal worker used to continue processing of atom. - */ -void kbase_sync_fence_wait_worker(struct work_struct *data); - -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG -/** - * kbase_sync_fence_in_dump() - Trigger a debug dump of atoms input fence state - * - * @katom: Atom to trigger fence debug dump for - */ -void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); -#endif -#endif /* !MALI_USE_CSF */ - #endif /* MALI_KBASE_SYNC_H */ diff --git a/drivers/gpu/arm/valhall/mali_kbase_sync_common.c b/drivers/gpu/arm/valhall/mali_kbase_sync_common.c index 5ee7fc3ce72b..05b896f4616f 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_sync_common.c +++ b/drivers/gpu/arm/valhall/mali_kbase_sync_common.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2012-2016, 2018-2021 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -29,16 +29,6 @@ #include "mali_kbase.h" #include "mali_kbase_sync.h" -#if !MALI_USE_CSF -void kbase_sync_fence_wait_worker(struct work_struct *data) -{ - struct kbase_jd_atom *katom; - - katom = container_of(data, struct kbase_jd_atom, work); - kbase_soft_event_wait_callback(katom); -} -#endif /* !MALI_USE_CSF */ - const char *kbase_sync_status_string(int status) { if (status == 0) diff --git a/drivers/gpu/arm/valhall/mali_kbase_sync_file.c b/drivers/gpu/arm/valhall/mali_kbase_sync_file.c index 38cd63d7a15b..940bcd034f58 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_sync_file.c +++ b/drivers/gpu/arm/valhall/mali_kbase_sync_file.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -52,66 +52,6 @@ int kbase_sync_fence_stream_create(const char *name, int *const out_fd) return 0; } -#if !MALI_USE_CSF -int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) -{ - struct dma_fence *fence; - struct sync_file *sync_file; - int fd; - - CSTD_UNUSED(stream_fd); - - fence = kbase_fence_out_new(katom); - if (!fence) - return -ENOMEM; - -#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) - /* Take an extra reference to the fence on behalf of the sync_file. - * This is only needed on older kernels where sync_file_create() - * does not take its own reference. This was changed in v4.9.68, - * where sync_file_create() now takes its own reference. - */ - dma_fence_get(fence); -#endif - - /* create a sync_file fd representing the fence */ - sync_file = sync_file_create(fence); - if (!sync_file) { -#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) - dma_fence_put(fence); -#endif - kbase_fence_out_remove(katom); - return -ENOMEM; - } - - fd = get_unused_fd_flags(O_CLOEXEC); - if (fd < 0) { - fput(sync_file->file); - kbase_fence_out_remove(katom); - return fd; - } - - fd_install((unsigned int)fd, sync_file->file); - - return fd; -} - -int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) -{ - struct dma_fence *fence = sync_file_get_fence(fd); - - lockdep_assert_held(&katom->kctx->jctx.lock); - - if (!fence) - return -ENOENT; - - kbase_fence_fence_in_set(katom, fence); - katom->dma_fence.fence_cb_added = false; - - return 0; -} -#endif /* !MALI_USE_CSF */ - int kbase_sync_fence_validate(int fd) { struct dma_fence *fence = sync_file_get_fence(fd); @@ -124,175 +64,6 @@ int kbase_sync_fence_validate(int fd) return 0; } -#if !MALI_USE_CSF -enum base_jd_event_code kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) -{ - int res; - - if (!kbase_fence_out_is_ours(katom)) { - /* Not our fence */ - return BASE_JD_EVENT_JOB_CANCELLED; - } - - res = kbase_fence_out_signal(katom, result); - if (unlikely(res < 0)) { - dev_warn(katom->kctx->kbdev->dev, "fence_signal() failed with %d\n", res); - } - - kbase_sync_fence_out_remove(katom); - - return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; -} - -static void kbase_fence_wait_callback(struct dma_fence *fence, struct dma_fence_cb *cb) -{ - struct kbase_jd_atom *katom = container_of(cb, struct kbase_jd_atom, dma_fence.fence_cb); - struct kbase_context *kctx = katom->kctx; - - CSTD_UNUSED(fence); - - /* Cancel atom if fence is erroneous */ - if (dma_fence_is_signaled(katom->dma_fence.fence_in) && -#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ - (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ - KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) - katom->dma_fence.fence_in->error < 0) -#else - katom->dma_fence.fence_in->status < 0) -#endif - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - /* To prevent a potential deadlock we schedule the work onto the - * job_done_wq workqueue - * - * The issue is that we may signal the timeline while holding - * kctx->jctx.lock and the callbacks are run synchronously from - * sync_timeline_signal. So we simply defer the work. - */ - INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); - queue_work(kctx->jctx.job_done_wq, &katom->work); -} - -int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) -{ - int err; - struct dma_fence *fence; - - lockdep_assert_held(&katom->kctx->jctx.lock); - - fence = katom->dma_fence.fence_in; - if (!fence) - return 0; /* no input fence to wait for, good to go! */ - - err = dma_fence_add_callback(fence, &katom->dma_fence.fence_cb, kbase_fence_wait_callback); - if (err == -ENOENT) { - int fence_status = dma_fence_get_status(fence); - - if (fence_status == 1) { - /* Fence is already signaled with no error. The completion - * for FENCE_WAIT softjob can be done right away. - */ - return 0; - } - - /* Fence shouldn't be in not signaled state */ - if (!fence_status) { - struct kbase_sync_fence_info info; - - kbase_sync_fence_in_info_get(katom, &info); - - dev_warn(katom->kctx->kbdev->dev, - "Unexpected status for fence %s of ctx:%d_%d atom:%d", info.name, - katom->kctx->tgid, katom->kctx->id, - kbase_jd_atom_id(katom->kctx, katom)); - } - - /* If fence is signaled with an error, then the FENCE_WAIT softjob is - * considered to be failed. - */ - } - - if (unlikely(err)) { - /* We should cause the dependent jobs in the bag to be failed. */ - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - /* The completion for FENCE_WAIT softjob can be done right away. */ - return 0; - } - - /* Callback was successfully installed */ - katom->dma_fence.fence_cb_added = true; - - /* Completion to be done later by callback/worker */ - return 1; -} - -void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&katom->kctx->jctx.lock); - - if (katom->dma_fence.fence_cb_added) { - if (!dma_fence_remove_callback(katom->dma_fence.fence_in, - &katom->dma_fence.fence_cb)) { - /* The callback is already removed so leave the cleanup - * for kbase_fence_wait_callback. - */ - return; - } - } else { - struct kbase_sync_fence_info info; - - kbase_sync_fence_in_info_get(katom, &info); - dev_warn(katom->kctx->kbdev->dev, - "Callback was not added earlier for fence %s of ctx:%d_%d atom:%d", - info.name, katom->kctx->tgid, katom->kctx->id, - kbase_jd_atom_id(katom->kctx, katom)); - } - - /* Wait was cancelled - zap the atoms */ - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - kbasep_remove_waiting_soft_job(katom); - kbase_finish_soft_job(katom); - - if (kbase_jd_done_nolock(katom, true)) - kbase_js_sched_all(katom->kctx->kbdev); -} - -void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom) -{ - kbase_fence_out_remove(katom); -} - -void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&katom->kctx->jctx.lock); - - if (katom->dma_fence.fence_cb_added) { - bool removed = dma_fence_remove_callback(katom->dma_fence.fence_in, - &katom->dma_fence.fence_cb); - - /* Here it is expected that the callback should have already been removed - * previously either by kbase_sync_fence_in_cancel_wait() or when the fence - * was signaled and kbase_sync_fence_wait_worker() was called. - */ - if (removed) { - struct kbase_sync_fence_info info; - - kbase_sync_fence_in_info_get(katom, &info); - dev_warn( - katom->kctx->kbdev->dev, - "Callback was not removed earlier for fence %s of ctx:%d_%d atom:%d", - info.name, katom->kctx->tgid, katom->kctx->id, - kbase_jd_atom_id(katom->kctx, katom)); - } - } - - kbase_fence_in_remove(katom); - katom->dma_fence.fence_cb_added = false; -} -#endif /* !MALI_USE_CSF */ - void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_info *info) { int status; @@ -318,43 +89,3 @@ void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_ scnprintf(info->name, sizeof(info->name), "%llu#%llu", fence->context, fence->seqno); #endif } - -#if !MALI_USE_CSF -int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info) -{ - struct dma_fence *fence; - - fence = kbase_fence_in_get(katom); - if (!fence) - return -ENOENT; - - kbase_sync_fence_info_get(fence, info); - - kbase_fence_put(fence); - - return 0; -} - -int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info) -{ - struct dma_fence *fence; - - fence = kbase_fence_out_get(katom); - if (!fence) - return -ENOENT; - - kbase_sync_fence_info_get(fence, info); - - kbase_fence_put(fence); - - return 0; -} - -#ifdef CONFIG_MALI_VALHALL_FENCE_DEBUG -void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) -{ - /* Not implemented */ - CSTD_UNUSED(katom); -} -#endif -#endif /* !MALI_USE_CSF*/ diff --git a/drivers/gpu/arm/valhall/mali_linux_trace.h b/drivers/gpu/arm/valhall/mali_linux_trace.h index 4dd5f437dad2..1b73153f6c9d 100644 --- a/drivers/gpu/arm/valhall/mali_linux_trace.h +++ b/drivers/gpu/arm/valhall/mali_linux_trace.h @@ -137,7 +137,6 @@ TRACE_EVENT(mali_total_alloc_pages_change, TP_PROTO(u32 gpu_id, s64 event_id), __print_symbolic(KBASE_MMU_FAULT_STATUS_ACCESS(status), \ KBASE_MMU_FAULT_ACCESS_SYMBOLIC_STRINGS) -#if MALI_USE_CSF #define KBASE_MMU_FAULT_CODE_VALID(code) \ ((code >= 0xC0 && code <= 0xEB) && (!(code >= 0xC5 && code <= 0xC7)) && \ (!(code >= 0xCC && code <= 0xD8)) && (!(code >= 0xDC && code <= 0xDF)) && \ @@ -148,20 +147,6 @@ TRACE_EVENT(mali_total_alloc_pages_change, TP_PROTO(u32 gpu_id, s64 event_id), { 0xD8, "ACCESS_FLAG_" }, { 0xE0, "ADDRESS_SIZE_FAULT_IN" }, \ { 0xE4, "ADDRESS_SIZE_FAULT_OUT" }, \ { 0xE8, "MEMORY_ATTRIBUTES_FAULT_" }) -#else /* MALI_USE_CSF */ -#define KBASE_MMU_FAULT_CODE_VALID(code) \ - ((code >= 0xC0 && code <= 0xEF) && (!(code >= 0xC5 && code <= 0xC6)) && \ - (!(code >= 0xCC && code <= 0xCF)) && (!(code >= 0xD4 && code <= 0xD7)) && \ - (!(code >= 0xDC && code <= 0xDF))) -#define KBASE_MMU_FAULT_CODE_SYMBOLIC_STRINGS \ - _ENSURE_PARENTHESIS({ 0xC0, "TRANSLATION_FAULT_" }, \ - { 0xC4, "TRANSLATION_FAULT(_7==_IDENTITY)_" }, \ - { 0xC8, "PERMISSION_FAULT_" }, { 0xD0, "TRANSTAB_BUS_FAULT_" }, \ - { 0xD8, "ACCESS_FLAG_" }, { 0xE0, "ADDRESS_SIZE_FAULT_IN" }, \ - { 0xE4, "ADDRESS_SIZE_FAULT_OUT" }, \ - { 0xE8, "MEMORY_ATTRIBUTES_FAULT_" }, \ - { 0xEC, "MEMORY_ATTRIBUTES_NONCACHEABLE_" }) -#endif /* MALI_USE_CSF */ #endif /* __TRACE_MALI_MMU_HELPERS */ /* trace_mali_mmu_page_fault_extra_grow @@ -255,35 +240,6 @@ DEFINE_EVENT_PRINT(mali_jit_softjob_template, mali_jit_free, TP_printk("start=0x%llx va_pages=0x%zx backed_size=0x%zx", __entry->start_addr, __entry->nr_pages, __entry->backed_pages)); -#if !MALI_USE_CSF -#if MALI_JIT_PRESSURE_LIMIT_BASE -/* trace_mali_jit_report - * - * Tracepoint about the GPU data structure read to form a just-in-time memory - * allocation report, and its calculated physical page usage - */ -TRACE_EVENT(mali_jit_report, - TP_PROTO(struct kbase_jd_atom *katom, struct kbase_va_region *reg, unsigned int id_idx, - u64 read_val, u64 used_pages), - TP_ARGS(katom, reg, id_idx, read_val, used_pages), - TP_STRUCT__entry(__field(u64, start_addr) __field(u64, read_val) - __field(u64, used_pages) __field(unsigned long, flags) - __field(u8, id_idx) __field(u8, jit_id)), - TP_fast_assign(__entry->start_addr = ((u64)reg->start_pfn) << PAGE_SHIFT; - __entry->read_val = read_val; __entry->used_pages = used_pages; - __entry->flags = reg->flags; __entry->id_idx = id_idx; - __entry->jit_id = katom->jit_ids[id_idx];), - TP_printk("start=0x%llx jit_ids[%u]=%u read_type='%s' read_val=0x%llx used_pages=%llu", - __entry->start_addr, __entry->id_idx, __entry->jit_id, - __print_symbolic(__entry->flags, { 0, "address" }, - { KBASE_REG_TILER_ALIGN_TOP, "address with align" }, - { KBASE_REG_HEAP_INFO_IS_SIZE, "size" }, - { KBASE_REG_HEAP_INFO_IS_SIZE | KBASE_REG_TILER_ALIGN_TOP, - "size with align (invalid)" }), - __entry->read_val, __entry->used_pages)); -#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ -#endif /* !MALI_USE_CSF */ - TRACE_DEFINE_ENUM(KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); #if MALI_JIT_PRESSURE_LIMIT_BASE /* trace_mali_jit_report_pressure diff --git a/drivers/gpu/arm/valhall/mmu/Kbuild b/drivers/gpu/arm/valhall/mmu/Kbuild index bedae1d67275..ac726539f4b1 100644 --- a/drivers/gpu/arm/valhall/mmu/Kbuild +++ b/drivers/gpu/arm/valhall/mmu/Kbuild @@ -23,13 +23,6 @@ valhall_kbase-y += \ mmu/mali_kbase_mmu_hw_direct.o \ mmu/mali_kbase_mmu_faults_decoder_luts.o \ mmu/mali_kbase_mmu_faults_decoder.o \ - mmu/mali_kbase_mmu_mode_aarch64.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += mmu/backend/mali_kbase_mmu_csf.o \ + mmu/mali_kbase_mmu_mode_aarch64.o \ + mmu/backend/mali_kbase_mmu_csf.o \ mmu/backend/mali_kbase_mmu_faults_decoder_luts_csf.o -else - valhall_kbase-y += mmu/backend/mali_kbase_mmu_jm.o \ - mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.o - -endif diff --git a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c b/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c deleted file mode 100644 index a053a93978b5..000000000000 --- a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.c +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Base kernel MMU faults decoder for Job Manager GPUs. - */ - -#include - -#define GPU_ID_ARCH_ID_MAJOR_GET(gpu_id) ((gpu_id >> 16) & 0xFF) -#define GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id) (gpu_id & 0xFFFF) -#define NELEMS(s) (sizeof(s) / sizeof((s)[0])) - -struct decode_lut_element { - u16 arch_minor_rev; - u16 key; - const char *text; -}; - -static const char *decode_lut_element_lookup(u16 arch_minor_rev, u16 key, - struct decode_lut_element *decode_element_lut, - unsigned int lut_len) -{ - struct decode_lut_element *p; - - for (p = decode_element_lut; p < decode_element_lut + lut_len; p++) { - if (p->key == key && - (p->arch_minor_rev == 0xffff || p->arch_minor_rev == arch_minor_rev)) - break; - } - if (p < decode_element_lut + lut_len) - return p->text; - else - return "unknown"; -} - -/* Auto-generated code: DO NOT MODIFY! */ - -static struct decode_lut_element lut_fault_source_jm_t_major_9[] = { - { 0xFFFF, 0, "js" }, - { 0xFFFF, 1, "pcm" }, -}; - -const char *decode_fault_source_jm_t(u16 idx, u32 gpu_id) -{ - u16 min_rev = GPU_ID_ARCH_ID_MINOR_AND_REV_GET(gpu_id); - const char *ret = "unknown"; - - switch (GPU_ID_ARCH_ID_MAJOR_GET(gpu_id)) { - case 9: - ret = decode_lut_element_lookup(min_rev, idx, lut_fault_source_jm_t_major_9, - NELEMS(lut_fault_source_jm_t_major_9)); - break; - } - return ret; -} diff --git a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h b/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h deleted file mode 100644 index f686e555d86a..000000000000 --- a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_faults_decoder_luts_jm.h +++ /dev/null @@ -1,37 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_JM_H_ -#define _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_JM_H_ -#include - -/** - * decode_fault_source_jm_t() - Get internal requester of a - * fault in a human readable format. - * - * @idx: Internal requester part of SOURCE_ID field of the fault. - * @gpu_id: GPU id composed of arch_major << 16 | arch_minor << 8 | arch_rev. - * - * Return: Internal requester of a fault in a human readable format for a JM core. - */ -const char *decode_fault_source_jm_t(u16 idx, u32 gpu_id); - -#endif /* _MALI_KBASE_MMU_FAULTS_DECODER_LUTS_JM_H_ */ diff --git a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_jm.c b/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_jm.c deleted file mode 100644 index 2ff0c1c0eae8..000000000000 --- a/drivers/gpu/arm/valhall/mmu/backend/mali_kbase_mmu_jm.c +++ /dev/null @@ -1,418 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/** - * DOC: Base kernel MMU management specific for Job Manager GPU. - */ - -#include -#include -#include -#include -#include -#include -#include - -void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, struct kbase_mmu_setup *const setup) -{ - /* Set up the required caching policies at the correct indices - * in the memattr register. - */ - setup->memattr = - (KBASE_MEMATTR_IMPL_DEF_CACHE_POLICY - << (KBASE_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | - (KBASE_MEMATTR_FORCE_TO_CACHE_ALL << (KBASE_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | - (KBASE_MEMATTR_WRITE_ALLOC << (KBASE_MEMATTR_INDEX_WRITE_ALLOC * 8)) | - (KBASE_MEMATTR_AARCH64_OUTER_IMPL_DEF << (KBASE_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | - (KBASE_MEMATTR_AARCH64_OUTER_WA << (KBASE_MEMATTR_INDEX_OUTER_WA * 8)) | - (KBASE_MEMATTR_AARCH64_NON_CACHEABLE << (KBASE_MEMATTR_INDEX_NON_CACHEABLE * 8)); - - setup->transtab = (u64)mmut->pgd & AS_TRANSTAB_BASE_MASK; - setup->transcfg = AS_TRANSCFG_MODE_SET(0ULL, AS_TRANSCFG_MODE_AARCH64_4K); -} - -void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbase_as *as, - struct kbase_fault *fault) -{ - struct kbase_device *const kbdev = kctx->kbdev; - const u32 status = fault->status; - const u32 exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); - const u32 access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); - const u32 source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); - unsigned int const as_no = as->number; - unsigned long flags; - const uintptr_t fault_addr = fault->addr; - - /* terminal fault, print info about the fault */ - dev_err(kbdev->dev, - "GPU bus fault in AS%u at PA %pK\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" - "pid: %d\n", - as_no, (void *)fault_addr, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(kbdev, access_type), source_id, - FAULT_SOURCE_ID_CORE_ID_GET(source_id), FAULT_SOURCE_ID_UTLB_ID_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), - kctx->pid); - - /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter - * dumping AS transaction begin - */ - mutex_lock(&kbdev->mmu_hw_mutex); - - /* Set the MMU into unmapped mode */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_mmu_disable(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - mutex_unlock(&kbdev->mmu_hw_mutex); - /* AS transaction end */ - - kbase_mmu_hw_clear_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); - kbase_mmu_hw_enable_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); - -} - -/* - * The caller must ensure it's retained the ctx to prevent it from being - * scheduled out whilst it's being worked on. - */ -void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as *as, - const char *reason_str, struct kbase_fault *fault) -{ - unsigned long flags; - struct kbase_device *kbdev = kctx->kbdev; - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - unsigned int as_no = as->number; - - /* Make sure the context was active */ - if (WARN_ON(atomic_read(&kctx->refcount) <= 0)) - return; - - if (!kbase_ctx_flag(kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) { - /* decode the fault status */ - const u32 status = fault->status; - const u32 exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); - const u32 access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); - const u32 source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); - /* terminal fault, print info about the fault */ - if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(9, 0)) { - dev_err(kbdev->dev, - "Unhandled Page fault in AS%u at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "pid: %d\n", - as_no, fault->addr, reason_str, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(kbdev, status), kctx->pid); - } else { - dev_err(kbdev->dev, - "Unhandled Page fault in AS%u at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" - "pid: %d\n", - as_no, fault->addr, reason_str, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(kbdev, status), source_id, - FAULT_SOURCE_ID_CORE_ID_GET(source_id), - FAULT_SOURCE_ID_UTLB_ID_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, - access_type), - kctx->pid); - } - } - - /* hardware counters dump fault handling */ - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && - (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING)) { - if ((fault->addr >= kbdev->hwcnt.addr) && - (fault->addr < (kbdev->hwcnt.addr + kbdev->hwcnt.addr_bytes))) - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; - } - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - /* Stop the kctx from submitting more jobs and cause it to be scheduled - * out/rescheduled - this will occur on releasing the context's refcount - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbasep_js_clear_submit_allowed(js_devdata, kctx); - - /* Kill any running jobs from the context. Submit is disallowed, so no - * more jobs from this context can appear in the job slots from this - * point on - */ - kbase_backend_jm_kill_running_jobs_from_kctx(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* AS transaction begin */ - mutex_lock(&kbdev->mmu_hw_mutex); - - /* switch to UNMAPPED mode, will abort all jobs and stop - * any hw counter dumping - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_mmu_disable(kctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - mutex_unlock(&kbdev->mmu_hw_mutex); - - /* AS transaction end */ - /* Clear down the fault */ - kbase_mmu_hw_clear_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); - kbase_mmu_hw_enable_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); - -} - -/** - * kbase_mmu_interrupt_process() - Process a bus or page fault. - * @kbdev: The kbase_device the fault happened on - * @kctx: The kbase_context for the faulting address space if one was - * found. - * @as: The address space that has the fault - * @fault: Data relating to the fault - * - * This function will process a fault on a specific address space - */ -static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, - struct kbase_as *as, struct kbase_fault *fault) -{ - unsigned long flags; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - dev_dbg(kbdev->dev, "Entering %s kctx %pK, as %pK\n", __func__, (void *)kctx, (void *)as); - - if (!kctx) { - if (kbase_as_has_bus_fault(as, fault)) { - dev_warn( - kbdev->dev, - "Bus error in AS%u at PA 0x%pK with no context present! Spurious IRQ or SW Design Error?\n", - as->number, (void *)(uintptr_t)fault->addr); - } else { - dev_warn( - kbdev->dev, - "Page fault in AS%u at VA 0x%016llx with no context present! Spurious IRQ or SW Design Error?\n", - as->number, fault->addr); - } - /* Since no ctx was found, the MMU must be disabled. */ - WARN_ON(as->current_setup.transtab); - - if (kbase_as_has_bus_fault(as, fault)) { - kbase_mmu_hw_clear_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); - kbase_mmu_hw_enable_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); - } else if (kbase_as_has_page_fault(as, fault)) { - kbase_mmu_hw_clear_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); - kbase_mmu_hw_enable_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); - } - - return; - } - - if (kbase_as_has_bus_fault(as, fault)) { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - - /* - * hw counters dumping in progress, signal the - * other thread that it failed - */ - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - if ((kbdev->hwcnt.kctx == kctx) && - (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING)) - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - /* - * Stop the kctx from submitting more jobs and cause it - * to be scheduled out/rescheduled when all references - * to it are released - */ - kbasep_js_clear_submit_allowed(js_devdata, kctx); - - dev_warn(kbdev->dev, "Bus error in AS%u at PA=0x%pK, IPA=0x%pK\n", as->number, - (void *)(uintptr_t)fault->addr, (void *)(uintptr_t)fault->extra_addr); - - /* - * We need to switch to UNMAPPED mode - but we do this in a - * worker so that we can sleep - */ - WARN_ON(!queue_work(as->pf_wq, &as->work_busfault)); - atomic_inc(&kbdev->faults_pending); - } else { - WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); - atomic_inc(&kbdev->faults_pending); - } - - dev_dbg(kbdev->dev, "Leaving %s kctx %pK, as %pK\n", __func__, (void *)kctx, (void *)as); -} - -static void validate_protected_page_fault(struct kbase_device *kbdev) -{ - /* GPUs which support (native) protected mode shall not report page - * fault addresses unless it has protected debug mode and protected - * debug mode is turned on - */ - u32 protected_debug_mode = 0; - - if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { - protected_debug_mode = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) & - GPU_STATUS_GPU_DBG_ENABLED; - } - - if (!protected_debug_mode) { - /* fault_addr should never be reported in protected mode. - * However, we just continue by printing an error message - */ - dev_err(kbdev->dev, "Fault address reported in protected mode\n"); - } -} - -void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) -{ - const int num_as = 16; - const int busfault_shift = MMU_PAGE_FAULT_FLAGS; - const int pf_shift = 0; - const unsigned long as_bit_mask = (1UL << num_as) - 1; - unsigned long flags; - u32 new_mask; - u32 tmp, bf_bits, pf_bits; - - dev_dbg(kbdev->dev, "Entering %s irq_stat %u\n", __func__, irq_stat); - /* bus faults */ - bf_bits = (irq_stat >> busfault_shift) & as_bit_mask; - /* page faults (note: Ignore ASes with both pf and bf) */ - pf_bits = ((irq_stat >> pf_shift) & as_bit_mask) & ~bf_bits; - - if (WARN_ON(kbdev == NULL)) - return; - - /* remember current mask */ - spin_lock_irqsave(&kbdev->mmu_mask_change, flags); - new_mask = kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK)); - /* mask interrupts for now */ - kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), 0); - spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); - - while (bf_bits | pf_bits) { - struct kbase_as *as; - unsigned int as_no; - struct kbase_context *kctx; - struct kbase_fault *fault; - - /* - * the while logic ensures we have a bit set, no need to check - * for not-found here - */ - as_no = (unsigned int)ffs((int)(bf_bits | pf_bits)) - 1; - as = &kbdev->as[as_no]; - - /* find the fault type */ - if (bf_bits & (1UL << as_no)) - fault = &as->bf_data; - else - fault = &as->pf_data; - - /* - * Refcount the kctx ASAP - it shouldn't disappear anyway, since - * Bus/Page faults _should_ only occur whilst jobs are running, - * and a job causing the Bus/Page fault shouldn't complete until - * the MMU is updated - */ - kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_no); - - /* find faulting address */ - fault->addr = kbase_reg_read64(kbdev, MMU_AS_OFFSET(as_no, FAULTADDRESS)); - /* Mark the fault protected or not */ - fault->protected_mode = kbdev->protected_mode; - - if (kbdev->protected_mode && fault->addr) { - /* check if address reporting is allowed */ - validate_protected_page_fault(kbdev); - } - - /* report the fault to debugfs */ - kbase_as_fault_debugfs_new(kbdev, as_no); - - /* record the fault status */ - fault->status = kbase_reg_read32(kbdev, MMU_AS_OFFSET(as_no, FAULTSTATUS)); - fault->extra_addr = kbase_reg_read64(kbdev, MMU_AS_OFFSET(as_no, FAULTEXTRA)); - - if (kbase_as_has_bus_fault(as, fault)) { - /* Mark bus fault as handled. - * Note that a bus fault is processed first in case - * where both a bus fault and page fault occur. - */ - bf_bits &= ~(1UL << as_no); - - /* remove the queued BF (and PF) from the mask */ - new_mask &= ~(MMU_BUS_ERROR(as_no) | MMU_PAGE_FAULT(as_no)); - } else { - /* Mark page fault as handled */ - pf_bits &= ~(1UL << as_no); - - /* remove the queued PF from the mask */ - new_mask &= ~MMU_PAGE_FAULT(as_no); - } - - /* Process the interrupt for this address space */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_mmu_interrupt_process(kbdev, kctx, as, fault); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } - - /* reenable interrupts */ - spin_lock_irqsave(&kbdev->mmu_mask_change, flags); - tmp = kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK)); - new_mask |= tmp; - kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), new_mask); - spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); - - dev_dbg(kbdev->dev, "Leaving %s irq_stat %u\n", __func__, irq_stat); -} - -int kbase_mmu_as_init(struct kbase_device *kbdev, unsigned int i) -{ - kbdev->as[i].number = i; - kbdev->as[i].bf_data.addr = 0ULL; - kbdev->as[i].pf_data.addr = 0ULL; - - kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%u", 0, 0, i); - if (!kbdev->as[i].pf_wq) - return -ENOMEM; - - INIT_WORK(&kbdev->as[i].work_pagefault, kbase_mmu_page_fault_worker); - INIT_WORK(&kbdev->as[i].work_busfault, kbase_mmu_bus_fault_worker); - - return 0; -} diff --git a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.c b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.c index 2e06ec90442e..7db1dfdc6d10 100644 --- a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.c +++ b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -43,9 +43,6 @@ #include #include #include -#if !MALI_USE_CSF -#include -#endif #include #include @@ -323,12 +320,8 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm /* Small wrapper function to factor out GPU-dependent context releasing */ static void release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) { -#if MALI_USE_CSF CSTD_UNUSED(kbdev); kbase_ctx_sched_release_ctx_lock(kctx); -#else /* MALI_USE_CSF */ - kbasep_js_runpool_release_ctx(kbdev, kctx); -#endif /* MALI_USE_CSF */ } /** @@ -357,7 +350,6 @@ static bool mmu_flush_cache_on_gpu_ctrl(struct kbase_device *kbdev) * * Issue a cache flush physical range command. */ -#if MALI_USE_CSF static void mmu_flush_pa_range(struct kbase_device *kbdev, phys_addr_t phys, size_t nr_bytes, enum kbase_mmu_op_type op) { @@ -378,7 +370,6 @@ static void mmu_flush_pa_range(struct kbase_device *kbdev, phys_addr_t phys, siz if (kbase_gpu_cache_flush_pa_range_and_busy_wait(kbdev, phys, nr_bytes, flush_op)) dev_err(kbdev->dev, "Flush for physical address range did not complete"); } -#endif /** * mmu_invalidate() - Perform an invalidate operation on MMU caches. @@ -500,13 +491,7 @@ static void mmu_flush_invalidate(struct kbase_device *kbdev, struct kbase_contex if (!kctx) { mmu_flush_invalidate_as(kbdev, &kbdev->as[as_nr], op_param); } else { -#if !MALI_USE_CSF - mutex_lock(&kbdev->js_data.queue_mutex); - ctx_is_in_runpool = kbase_ctx_sched_inc_refcount(kctx); - mutex_unlock(&kbdev->js_data.queue_mutex); -#else ctx_is_in_runpool = kbase_ctx_sched_inc_refcount_if_as_valid(kctx); -#endif /* !MALI_USE_CSF */ if (ctx_is_in_runpool) { KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); @@ -682,6 +667,12 @@ static bool kbase_mmu_handle_isolated_pgd_page(struct kbase_device *kbdev, return page_is_isolated; } +static struct kbase_mem_pool *kbase_mmu_get_pgd_pool(struct kbase_device *kbdev, + struct kbase_mmu_table *mmut) +{ + return (mmut->kctx) ? &mmut->kctx->pgd_mem_pool : &kbdev->pgd_mem_pool; +} + /** * kbase_mmu_free_pgd() - Free memory of the page directory * @@ -718,7 +709,7 @@ static void kbase_mmu_free_pgd(struct kbase_device *kbdev, struct kbase_mmu_tabl page_is_isolated = kbase_mmu_handle_isolated_pgd_page(kbdev, mmut, p); if (likely(!page_is_isolated)) { - kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], p, true); + kbase_mem_pool_free(kbase_mmu_get_pgd_pool(kbdev, mmut), p, true); kbase_mmu_account_freed_pgd(kbdev, mmut); } } @@ -810,82 +801,12 @@ static size_t reg_grow_calc_extra_pages(struct kbase_device *kbdev, struct kbase */ remainder = minimum_extra % multiple; -#if !MALI_USE_CSF - if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { - /* multiple is based from the top of the initial commit, which - * has been allocated in such a way that (start_pfn + - * initial_commit) is already aligned to multiple. Hence the - * pfn for the end of committed memory will also be aligned to - * multiple - */ - size_t initial_commit = reg->initial_commit; - - if (fault_rel_pfn < initial_commit) { - /* this case is just to catch in case it's been - * recommitted by userspace to be smaller than the - * initial commit - */ - minimum_extra = initial_commit - reg_current_size; - remainder = 0; - } else { - /* same as calculating - * (fault_rel_pfn - initial_commit + 1) - */ - size_t pages_after_initial = - minimum_extra + reg_current_size - initial_commit; - - remainder = pages_after_initial % multiple; - } - } -#endif /* !MALI_USE_CSF */ - if (remainder == 0) return minimum_extra; return minimum_extra + multiple - remainder; } -/** - * estimate_pool_space_required - Determine how much a pool should be grown by to support a future - * allocation - * @pool: The memory pool to check, including its linked pools - * @pages_required: Number of small pages require for the pool to support a future allocation - * - * The value returned is accounting for the size of @pool and the size of each memory pool linked to - * @pool. Hence, the caller should use @pool and (if not already satisfied) all its linked pools to - * allocate from. - * - * Note: this is only an estimate, because even during the calculation the memory pool(s) involved - * can be updated to be larger or smaller. Hence, the result is only a guide as to whether an - * allocation could succeed, or an estimate of the correct amount to grow the pool by. The caller - * should keep attempting an allocation and then re-growing with a new value queried form this - * function until the allocation succeeds. - * - * Return: an estimate of the amount of extra small pages in @pool that are required to satisfy an - * allocation, or 0 if @pool (including its linked pools) is likely to already satisfy the - * allocation. - */ -static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const size_t pages_required) -{ - size_t pages_still_required; - - for (pages_still_required = pages_required; pool != NULL && pages_still_required; - pool = pool->next_pool) { - size_t pool_size_small; - - kbase_mem_pool_lock(pool); - - pool_size_small = kbase_mem_pool_size(pool) << pool->order; - if (pool_size_small >= pages_still_required) - pages_still_required = 0; - else - pages_still_required -= pool_size_small; - - kbase_mem_pool_unlock(pool); - } - return pages_still_required; -} - /** * page_fault_try_alloc - Try to allocate memory from a context pool * @kctx: Context pointer @@ -898,11 +819,10 @@ static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const si * @fallback_to_small: Whether fallback to small pages or not * @prealloc_sas: Pointer to kbase_sub_alloc structures * - * This function will try to allocate as many pages as possible from the context pool, then if - * required will try to allocate the remaining pages from the device pool. + * This function will try to allocate as many pages as possible from the pool * - * This function will not allocate any new memory beyond that is already present in the context or - * device pools. This is because it is intended to be called whilst the thread has acquired the + * This function will not allocate any new memory beyond that is already present in the context. + * This is because it is intended to be called whilst the thread has acquired the * region list lock with kbase_gpu_vm_lock(), and a large enough memory allocation whilst that is * held could invoke the OoM killer and cause an effective deadlock with kbase_cpu_vm_close(). * @@ -915,152 +835,91 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg size_t new_pages, size_t *pages_to_grow, bool *grow_2mb_pool, bool fallback_to_small, struct kbase_sub_alloc **prealloc_sas) { - size_t total_gpu_pages_alloced = 0; - size_t total_cpu_pages_alloced = 0; - struct kbase_mem_pool *pool, *root_pool; + size_t gpu_pages_alloced = 0; + size_t cpu_pages_alloced = 0; + struct kbase_mem_pool *pool; bool alloc_failed = false; - size_t pages_still_required; - size_t total_mempools_free_small = 0; + size_t pool_size_small; + size_t pages_to_alloc_small_per_alloc; + struct tagged_addr *gpu_pages; lockdep_assert_held(&kctx->reg_lock); lockdep_assert_held(&kctx->mem_partials_lock); - if (WARN_ON(region->gpu_alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) { - /* Do not try to grow the memory pool */ - *pages_to_grow = 0; + *pages_to_grow = 0; + + if (WARN_ON(region->gpu_alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return false; - } if (kbase_is_large_pages_enabled() && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE && !fallback_to_small) { - root_pool = &kctx->mem_pools.large[region->gpu_alloc->group_id]; + pool = &kctx->mem_pools.large[region->gpu_alloc->group_id]; *grow_2mb_pool = true; } else { - root_pool = &kctx->mem_pools.small[region->gpu_alloc->group_id]; + pool = &kctx->mem_pools.small[region->gpu_alloc->group_id]; *grow_2mb_pool = false; } + pages_to_alloc_small_per_alloc = new_pages; if (region->gpu_alloc != region->cpu_alloc) - new_pages *= 2; + new_pages <<= 1; + kbase_mem_pool_lock(pool); /* Determine how many pages are in the pools before trying to allocate. * Don't attempt to allocate & free if the allocation can't succeed. */ - pages_still_required = estimate_pool_space_required(root_pool, new_pages); - - if (pages_still_required) { + pool_size_small = kbase_mem_pool_size(pool) << pool->order; + if (pool_size_small < new_pages) { /* Insufficient pages in pools. Don't try to allocate - just * request a grow. */ - *pages_to_grow = pages_still_required; - + *pages_to_grow = new_pages - pool_size_small; + kbase_mem_pool_unlock(pool); return false; } - /* Since we're not holding any of the mempool locks, the amount of memory in the pools may - * change between the above estimate and the actual allocation. - */ - pages_still_required = new_pages; - for (pool = root_pool; pool != NULL && pages_still_required; pool = pool->next_pool) { - size_t pool_size_small; - size_t pages_to_alloc_small; - size_t pages_to_alloc_small_per_alloc; + gpu_pages = kbase_alloc_phy_pages_helper_locked( + region->gpu_alloc, pool, pages_to_alloc_small_per_alloc, &prealloc_sas[0]); - kbase_mem_pool_lock(pool); + if (!gpu_pages) + alloc_failed = true; + else + gpu_pages_alloced = pages_to_alloc_small_per_alloc; - /* Allocate as much as possible from this pool*/ - pool_size_small = kbase_mem_pool_size(pool) << pool->order; - total_mempools_free_small += pool_size_small; - pages_to_alloc_small = MIN(pages_still_required, pool_size_small); - if (region->gpu_alloc == region->cpu_alloc) - pages_to_alloc_small_per_alloc = pages_to_alloc_small; + if (!alloc_failed && region->gpu_alloc != region->cpu_alloc) { + struct tagged_addr *cpu_pages = kbase_alloc_phy_pages_helper_locked( + region->cpu_alloc, pool, pages_to_alloc_small_per_alloc, &prealloc_sas[1]); + + if (!cpu_pages) + alloc_failed = true; else - pages_to_alloc_small_per_alloc = pages_to_alloc_small >> 1; - - if (pages_to_alloc_small) { - struct tagged_addr *gpu_pages = kbase_alloc_phy_pages_helper_locked( - region->gpu_alloc, pool, pages_to_alloc_small_per_alloc, - &prealloc_sas[0]); - - if (!gpu_pages) - alloc_failed = true; - else - total_gpu_pages_alloced += pages_to_alloc_small_per_alloc; - - if (!alloc_failed && region->gpu_alloc != region->cpu_alloc) { - struct tagged_addr *cpu_pages = kbase_alloc_phy_pages_helper_locked( - region->cpu_alloc, pool, pages_to_alloc_small_per_alloc, - &prealloc_sas[1]); - - if (!cpu_pages) - alloc_failed = true; - else - total_cpu_pages_alloced += pages_to_alloc_small_per_alloc; - } - } - - kbase_mem_pool_unlock(pool); - - if (alloc_failed) { - WARN_ON(!pages_still_required); - WARN_ON(pages_to_alloc_small >= pages_still_required); - WARN_ON(pages_to_alloc_small_per_alloc >= pages_still_required); - break; - } - - pages_still_required -= pages_to_alloc_small; + cpu_pages_alloced = pages_to_alloc_small_per_alloc; } - if (pages_still_required) { + kbase_mem_pool_unlock(pool); + + if (alloc_failed) { /* Allocation was unsuccessful. We have dropped the mem_pool lock after allocation, * so must in any case use kbase_free_phy_pages_helper() rather than * kbase_free_phy_pages_helper_locked() */ - if (total_gpu_pages_alloced > 0) - kbase_free_phy_pages_helper(region->gpu_alloc, total_gpu_pages_alloced); - if (region->gpu_alloc != region->cpu_alloc && total_cpu_pages_alloced > 0) - kbase_free_phy_pages_helper(region->cpu_alloc, total_cpu_pages_alloced); - - if (alloc_failed) { - /* Note that in allocating from the above memory pools, we always ensure - * never to request more than is available in each pool with the pool's - * lock held. Hence failing to allocate in such situations would be unusual - * and we should cancel the growth instead (as re-growing the memory pool - * might not fix the situation) - */ - dev_warn( - kctx->kbdev->dev, - "Page allocation failure of %zu pages: managed %zu pages, mempool (inc linked pools) had %zu pages available", - new_pages, total_gpu_pages_alloced + total_cpu_pages_alloced, - total_mempools_free_small); - *pages_to_grow = 0; - } else { - /* Tell the caller to try to grow the memory pool - * - * Freeing pages above may have spilled or returned them to the OS, so we - * have to take into account how many are still in the pool before giving a - * new estimate for growth required of the pool. We can just re-estimate a - * new value. - */ - pages_still_required = estimate_pool_space_required(root_pool, new_pages); - if (pages_still_required) { - *pages_to_grow = pages_still_required; - } else { - /* It's possible another thread could've grown the pool to be just - * big enough after we rolled back the allocation. Request at least - * one more page to ensure the caller doesn't fail the growth by - * conflating it with the alloc_failed case above - */ - *pages_to_grow = 1u; - } - } + if (gpu_pages_alloced > 0) + kbase_free_phy_pages_helper(region->gpu_alloc, gpu_pages_alloced); + if (region->gpu_alloc != region->cpu_alloc && cpu_pages_alloced > 0) + kbase_free_phy_pages_helper(region->cpu_alloc, cpu_pages_alloced); + /* Failing to allocate in such situation is unusual, + * because the pool was supposed to have enough memory pages, + * and we should cancel the growth as re-growing the memory pool + * might not fix the situation. + */ + dev_warn( + kctx->kbdev->dev, + "Page allocation failure of %zu pages: managed %zu pages, mempool had %zu pages available", + new_pages, gpu_pages_alloced + cpu_pages_alloced, pool_size_small); return false; } - /* Allocation was successful. No pages to grow, return success. */ - *pages_to_grow = 0; - return true; } @@ -1116,9 +975,6 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev); #if MALI_JIT_PRESSURE_LIMIT_BASE -#if !MALI_USE_CSF - mutex_lock(&kctx->jctx.lock); -#endif #endif /* check if we still have GPU */ @@ -1145,10 +1001,6 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_3: fallthrough; case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_4: -#if !MALI_USE_CSF - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_IDENTITY: -#endif /* need to check against the region to handle this one */ break; @@ -1162,23 +1014,6 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Permission failure", fault); goto fault_done; -#if !MALI_USE_CSF - case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_0: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_1: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_2: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSTAB_BUS_FAULT_3: - kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Translation table bus fault", - fault); - goto fault_done; -#endif - -#if !MALI_USE_CSF - case AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_0: - fallthrough; -#endif case AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_1: fallthrough; case AS_FAULTSTATUS_EXCEPTION_TYPE_ACCESS_FLAG_2: @@ -1188,19 +1023,8 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) dev_warn(kbdev->dev, "Access flag unexpectedly set"); goto fault_done; -#if MALI_USE_CSF case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN: fallthrough; -#else - case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN0: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN1: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN2: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_IN3: - fallthrough; -#endif case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT0: fallthrough; case AS_FAULTSTATUS_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT1: @@ -1218,16 +1042,6 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_2: fallthrough; case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_3: -#if !MALI_USE_CSF - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_0: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_1: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_2: - fallthrough; - case AS_FAULTSTATUS_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_NONCACHEABLE_3: -#endif kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Memory attributes fault", fault); goto fault_done; @@ -1557,9 +1371,6 @@ fault_done: kbase_jit_done_phys_increase(kctx, pages_trimmed); kbase_gpu_vm_unlock(kctx); } -#if !MALI_USE_CSF - mutex_unlock(&kctx->jctx.lock); -#endif #endif for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) @@ -1592,6 +1403,7 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_ u64 *page; struct page *p; phys_addr_t pgd; + struct kbase_mem_pool *pgd_mem_pool = kbase_mmu_get_pgd_pool(kbdev, mmut); lockdep_assert_held(&mmut->mmu_lock); @@ -1601,7 +1413,7 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_ return pgd; #endif - p = kbase_mem_pool_alloc(&kbdev->mem_pools.small[mmut->group_id]); + p = kbase_mem_pool_alloc(pgd_mem_pool); if (!p) return KBASE_INVALID_PHYSICAL_ADDRESS; @@ -1648,7 +1460,7 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_ return pgd; alloc_free: - kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], p, false); + kbase_mem_pool_free(pgd_mem_pool, p, false); return KBASE_INVALID_PHYSICAL_ADDRESS; } @@ -1940,11 +1752,7 @@ static void mmu_flush_invalidate_insert_pages(struct kbase_device *kbdev, op_param.kctx_id = mmut->kctx ? mmut->kctx->id : 0xFFFFFFFF; op_param.flush_skip_levels = pgd_level_to_skip_flush(dirty_pgds); -#if MALI_USE_CSF as_nr = mmut->kctx ? mmut->kctx->as_nr : MCU_AS_NR; -#else - WARN_ON(!mmut->kctx); -#endif /* MMU cache flush strategy depends on whether GPU control commands for * flushing physical address ranges are supported. The new physical pages @@ -2150,6 +1958,7 @@ static int mmu_insert_alloc_pgds(struct kbase_device *kbdev, struct kbase_mmu_ta { int err = 0; int i; + struct kbase_mem_pool *pgd_mem_pool = kbase_mmu_get_pgd_pool(kbdev, mmut); lockdep_assert_held(&mmut->mmu_lock); @@ -2162,8 +1971,7 @@ static int mmu_insert_alloc_pgds(struct kbase_device *kbdev, struct kbase_mmu_ta if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS) break; mutex_unlock(&mmut->mmu_lock); - err = kbase_mem_pool_grow(&kbdev->mem_pools.small[mmut->group_id], - (size_t)level_high, NULL); + err = kbase_mem_pool_grow(pgd_mem_pool, (size_t)level_high, NULL); mutex_lock(&mmut->mmu_lock); if (err) { dev_err(kbdev->dev, "%s: kbase_mem_pool_grow() returned error %d", @@ -2410,6 +2218,10 @@ static void kbase_mmu_progress_migration_on_insert(struct tagged_addr phys, if (!kbase_is_page_migration_enabled()) return; + /* Metadata not created */ + if (!page_md) + return; + spin_lock(&page_md->migrate_lock); /* If no GPU va region is given: the metadata provided are @@ -2490,10 +2302,15 @@ u64 kbase_mmu_create_ate(struct kbase_device *const kbdev, struct tagged_addr co { u64 entry; unsigned int pte_flags = 0; + const bool cpu_access = flags & (KBASE_REG_CPU_RD | KBASE_REG_CPU_WR); kbdev->mmu_mode->entry_set_ate(&entry, phy, flags, level); - if ((flags & KBASE_REG_GPU_CACHED) && !(flags & KBASE_REG_CPU_CACHED)) + /* Address-Table Entries (ATEs) that are GPU-cached but CPU uncached are flagged as a + * Mismatched Memory Attribute (MMA) violation, as per A5.3.2 of the AMBA protocol + * specification. Depending on the system, these might need special handling: + */ + if ((flags & KBASE_REG_GPU_CACHED) && cpu_access && !(flags & KBASE_REG_CPU_CACHED)) pte_flags |= BIT(MMA_VIOLATION); return kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, (unsigned int)group_id, @@ -2844,9 +2661,6 @@ KBASE_EXPORT_TEST_API(kbase_mmu_update); void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) { lockdep_assert_held(&kbdev->hwaccess_lock); -#if !MALI_USE_CSF - lockdep_assert_held(&kbdev->mmu_hw_mutex); -#endif kbdev->mmu_mode->disable_as(kbdev, as_nr); } @@ -2881,7 +2695,6 @@ void kbase_mmu_disable(struct kbase_context *kctx) op_param.kctx_id = kctx->id; op_param.mmu_sync_info = mmu_sync_info; -#if MALI_USE_CSF /* 0xF value used to prevent skipping of any levels when flushing */ if (mmu_flush_cache_on_gpu_ctrl(kbdev)) op_param.flush_skip_levels = pgd_level_to_skip_flush(0xF); @@ -2916,36 +2729,6 @@ void kbase_mmu_disable(struct kbase_context *kctx) dev_err(kbdev->dev, "Failed to unlock AS %d for ctx %d_%d", kctx->as_nr, kctx->tgid, kctx->id); } -#else - lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); - - CSTD_UNUSED(lock_err); - - /* - * The address space is being disabled, drain all knowledge of it out - * from the caches as pages and page tables might be freed after this. - * - * The job scheduler code will already be holding the locks and context - * so just do the flush. - */ - flush_err = kbase_mmu_hw_do_flush(kbdev, &kbdev->as[kctx->as_nr], &op_param); - if (flush_err) { - dev_err(kbdev->dev, - "Flush for GPU page table update did not complete to disable AS %d for ctx %d_%d", - kctx->as_nr, kctx->tgid, kctx->id); - /* GPU reset would have been triggered by the flush function */ - } - - kbdev->mmu_mode->disable_as(kbdev, kctx->as_nr); - - /* - * JM GPUs has some L1 read only caches that need to be invalidated - * with START_FLUSH configuration. Purge the MMU disabled kctx from - * the slot_rb tracking field so such invalidation is performed when - * a new katom is executed on the affected slots. - */ - kbase_backend_slot_kctx_purge_locked(kbdev, kctx); -#endif } KBASE_EXPORT_TEST_API(kbase_mmu_disable); @@ -3049,9 +2832,7 @@ static void mmu_flush_invalidate_teardown_pages(struct kbase_device *kbdev, } else if (op_param->op == KBASE_MMU_OP_FLUSH_MEM) { /* Full cache flush through the GPU_CONTROL */ mmu_flush_invalidate_on_gpu_ctrl(kbdev, kctx, as_nr, op_param); - } -#if MALI_USE_CSF - else { + } else { /* Partial GPU cache flush of the pages that were unmapped */ unsigned long irq_flags; unsigned int i; @@ -3067,10 +2848,6 @@ static void mmu_flush_invalidate_teardown_pages(struct kbase_device *kbdev, spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); } } -#else - CSTD_UNUSED(phys); - CSTD_UNUSED(phys_page_nr); -#endif } static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, @@ -3506,13 +3283,6 @@ static int kbase_mmu_update_pages_common(struct kbase_device *kbdev, struct kbas const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC; int as_nr; -#if !MALI_USE_CSF - if (unlikely(kctx == NULL)) - return -EINVAL; - - as_nr = kctx->as_nr; - mmut = &kctx->mmu; -#else if (kctx) { mmut = &kctx->mmu; as_nr = kctx->as_nr; @@ -3520,7 +3290,6 @@ static int kbase_mmu_update_pages_common(struct kbase_device *kbdev, struct kbas mmut = &kbdev->csf.mcu_mmu; as_nr = MCU_AS_NR; } -#endif err = kbase_mmu_update_pages_no_flush(kbdev, mmut, vpfn, phys, nr, flags, group_id, &dirty_pgds); @@ -3551,13 +3320,11 @@ int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, struct tagged_a return kbase_mmu_update_pages_common(kctx->kbdev, kctx, vpfn, phys, nr, flags, group_id); } -#if MALI_USE_CSF int kbase_mmu_update_csf_mcu_pages(struct kbase_device *kbdev, u64 vpfn, struct tagged_addr *phys, size_t nr, unsigned long flags, int const group_id) { return kbase_mmu_update_pages_common(kbdev, NULL, vpfn, phys, nr, flags, group_id); } -#endif /* MALI_USE_CSF */ static void mmu_page_migration_transaction_begin(struct kbase_device *kbdev) { @@ -3753,13 +3520,11 @@ static int mmu_migrate_pgd_sub_page(phys_addr_t old_pgd_phys, phys_addr_t new_pg ret = kbase_mmu_hw_do_lock(kbdev, as, &op_param); if (!ret) { -#if MALI_USE_CSF if (mmu_flush_cache_on_gpu_ctrl(kbdev)) ret = kbase_gpu_cache_flush_pa_range_and_busy_wait( kbdev, old_pgd_phys, GPU_PAGE_SIZE, GPU_COMMAND_FLUSH_PA_RANGE_CLN_INV_L2_LSC); else -#endif ret = kbase_gpu_cache_flush_and_busy_wait( kbdev, GPU_COMMAND_CACHE_CLN_INV_L2_LSC); } @@ -4159,13 +3924,11 @@ int kbase_mmu_migrate_data_page(struct tagged_addr old_phys, struct tagged_addr ret = kbase_mmu_hw_do_lock(kbdev, as, &op_param); if (!ret) { -#if MALI_USE_CSF if (mmu_flush_cache_on_gpu_ctrl(kbdev)) ret = kbase_gpu_cache_flush_pa_range_and_busy_wait( kbdev, as_phys_addr_t(old_phys), PAGE_SIZE, GPU_COMMAND_FLUSH_PA_RANGE_CLN_INV_L2_LSC); else -#endif ret = kbase_gpu_cache_flush_and_busy_wait( kbdev, GPU_COMMAND_CACHE_CLN_INV_L2_LSC); } @@ -4411,9 +4174,10 @@ int kbase_mmu_init(struct kbase_device *const kbdev, struct kbase_mmu_table *con */ while (mmut->pgd == KBASE_INVALID_PHYSICAL_ADDRESS) { int err; + struct kbase_mem_pool *pgd_mem_pool = kbase_mmu_get_pgd_pool(kbdev, mmut); - err = kbase_mem_pool_grow(&kbdev->mem_pools.small[mmut->group_id], - MIDGARD_MMU_BOTTOMLEVEL, kctx ? kctx->task : NULL); + err = kbase_mem_pool_grow(pgd_mem_pool, MIDGARD_MMU_BOTTOMLEVEL, + kctx ? kctx->task : NULL); if (err) { kbase_mmu_term(kbdev, mmut); return -ENOMEM; @@ -4454,7 +4218,6 @@ void kbase_mmu_as_term(struct kbase_device *kbdev, unsigned int i) void kbase_mmu_flush_pa_range(struct kbase_device *kbdev, struct kbase_context *kctx, phys_addr_t phys, size_t size, enum kbase_mmu_op_type flush_op) { -#if MALI_USE_CSF unsigned long irq_flags; spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); @@ -4462,13 +4225,6 @@ void kbase_mmu_flush_pa_range(struct kbase_device *kbdev, struct kbase_context * kbdev->pm.backend.gpu_ready && (!kctx || kctx->as_nr >= 0)) mmu_flush_pa_range(kbdev, phys, size, KBASE_MMU_OP_FLUSH_PT); spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -#else - CSTD_UNUSED(kbdev); - CSTD_UNUSED(kctx); - CSTD_UNUSED(phys); - CSTD_UNUSED(size); - CSTD_UNUSED(flush_op); -#endif } #ifdef CONFIG_MALI_VECTOR_DUMP @@ -4656,22 +4412,11 @@ void kbase_mmu_bus_fault_worker(struct work_struct *data) return; } -#if MALI_USE_CSF /* Before the GPU power off, wait is done for the completion of * in-flight MMU fault work items. So GPU is expected to remain * powered up whilst the bus fault handling is being done. */ kbase_gpu_report_bus_fault_and_kill(kctx, faulting_as, fault); -#else - /* NOTE: If GPU already powered off for suspend, - * we don't need to switch to unmapped - */ - if (!kbase_pm_context_active_handle_suspend(kbdev, - KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { - kbase_gpu_report_bus_fault_and_kill(kctx, faulting_as, fault); - kbase_pm_context_idle(kbdev); - } -#endif release_ctx(kbdev, kctx); diff --git a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.h b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.h index f0a418002247..c952346efa21 100644 --- a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.h +++ b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu.h @@ -215,7 +215,6 @@ int kbase_mmu_teardown_imported_pages(struct kbase_device *kbdev, struct kbase_m int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, struct tagged_addr *phys, size_t nr, unsigned long flags, int const group_id); -#if MALI_USE_CSF /** * kbase_mmu_update_csf_mcu_pages - Update MCU mappings with changes of phys and flags * @@ -233,7 +232,6 @@ int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, struct tagged_a */ int kbase_mmu_update_csf_mcu_pages(struct kbase_device *kbdev, u64 vpfn, struct tagged_addr *phys, size_t nr, unsigned long flags, int const group_id); -#endif /** * kbase_mmu_migrate_data_page - Migrate GPU mappings and content of data pages between memory pages diff --git a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_faults_decoder.c b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_faults_decoder.c index 548d88cf216e..16057a29bb14 100644 --- a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_faults_decoder.c +++ b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_faults_decoder.c @@ -25,11 +25,7 @@ #include #include -#if MALI_USE_CSF #include -#else -#include -#endif #include #include @@ -76,7 +72,6 @@ const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbde } else return "Load/store cache"; } else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "tiler")) { -#if MALI_USE_CSF if (utlb_id == 0) { if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) return decode_fault_source_tiler_r_t( @@ -86,25 +81,14 @@ const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbde ir, kbdev->gpu_props.gpu_id.arch_id); } else return "The polygon list writer. No further details."; -#else - return (utlb_id == 0) ? "Anything other than the polygon list writer" : - "The polygon list writer"; -#endif - } -#if MALI_USE_CSF - else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "csf")) { + } else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "csf")) { if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) return decode_fault_source_csf_r_t(ir, kbdev->gpu_props.gpu_id.arch_id); else return decode_fault_source_csf_w_t(ir, kbdev->gpu_props.gpu_id.arch_id); - } -#else - else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "jm")) - return decode_fault_source_jm_t(ir, kbdev->gpu_props.gpu_id.arch_id); -#endif - else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "I2c") || - !strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "memsys") || - !strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "mmu")) { + } else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "I2c") || + !strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "memsys") || + !strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "mmu")) { return "Not used"; } diff --git a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_hw_direct.c index e531444ccb24..6051550b1ef7 100644 --- a/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_hw_direct.c +++ b/drivers/gpu/arm/valhall/mmu/mali_kbase_mmu_hw_direct.c @@ -29,7 +29,6 @@ #include #include -#if MALI_USE_CSF /** * mmu_has_flush_skip_pgd_levels() - Check if the GPU has the feature * AS_LOCKADDR_FLUSH_SKIP_LEVELS @@ -45,7 +44,6 @@ static bool mmu_has_flush_skip_pgd_levels(struct kbase_gpu_props const *gpu_prop { return gpu_props->gpu_id.arch_id >= GPU_ID_ARCH_MAKE(12, 0, 4); } -#endif /** * lock_region() - Generate lockaddr to lock memory region in MMU @@ -145,11 +143,9 @@ static int lock_region(struct kbase_gpu_props const *gpu_props, u64 *lockaddr, *lockaddr = lockaddr_base & ~((1ull << lockaddr_size_log2) - 1); *lockaddr |= lockaddr_size_log2 - 1; -#if MALI_USE_CSF if (mmu_has_flush_skip_pgd_levels(gpu_props)) *lockaddr = AS_LOCKADDR_FLUSH_SKIP_LEVELS_SET(*lockaddr, op_param->flush_skip_levels); -#endif return 0; } @@ -210,7 +206,6 @@ static int write_cmd(struct kbase_device *kbdev, unsigned int as_nr, u32 cmd) return status; } -#if MALI_USE_CSF static int wait_l2_power_trans_complete(struct kbase_device *kbdev) { u32 val; @@ -316,7 +311,6 @@ static int apply_hw_issue_GPU2019_3901_wa(struct kbase_device *kbdev, u32 *mmu_c return ret; } #endif /* !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) */ -#endif /* MALI_USE_CSF */ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) { @@ -324,9 +318,6 @@ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) u64 transcfg = 0; lockdep_assert_held(&kbdev->hwaccess_lock); -#if !MALI_USE_CSF - lockdep_assert_held(&kbdev->mmu_hw_mutex); -#endif transcfg = current_setup->transcfg; @@ -343,18 +334,19 @@ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) transcfg = AS_TRANSCFG_PTW_SH_SET(transcfg, AS_TRANSCFG_PTW_SH_OUTER_SHAREABLE); } + kbase_reg_write64(kbdev, MMU_AS_OFFSET(as->number, TRANSCFG), transcfg); kbase_reg_write64(kbdev, MMU_AS_OFFSET(as->number, TRANSTAB), current_setup->transtab); kbase_reg_write64(kbdev, MMU_AS_OFFSET(as->number, MEMATTR), current_setup->memattr); KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(kbdev, as, current_setup->transtab, current_setup->memattr, transcfg); + KBASE_TLSTREAM_JD_AS_INFO(kbdev, as->number, current_setup->transtab, + current_setup->memattr, transcfg); write_cmd(kbdev, as->number, AS_COMMAND_COMMAND_UPDATE); -#if MALI_USE_CSF /* Wait for UPDATE command to complete */ wait_ready(kbdev, as->number); -#endif } /** @@ -465,6 +457,7 @@ int kbase_mmu_hw_do_unlock_no_addr(struct kbase_device *kbdev, struct kbase_as * if (WARN_ON(kbdev == NULL) || WARN_ON(as == NULL)) return -EINVAL; + ret = write_cmd(kbdev, as->number, AS_COMMAND_COMMAND_UNLOCK); /* Wait for UNLOCK command to complete */ @@ -491,6 +484,7 @@ int kbase_mmu_hw_do_unlock(struct kbase_device *kbdev, struct kbase_as *as, if (WARN_ON(kbdev == NULL) || WARN_ON(as == NULL)) return -EINVAL; + ret = mmu_hw_set_lock_addr(kbdev, as->number, &lock_addr, op_param); if (!ret) @@ -528,7 +522,7 @@ int kbase_mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, if (ret) return ret; -#if MALI_USE_CSF && !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) +#if !IS_ENABLED(CONFIG_MALI_VALHALL_NO_MALI) /* WA for the KBASE_HW_ISSUE_GPU2019_3901. */ if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901) && mmu_cmd == AS_COMMAND_COMMAND_FLUSH_MEM) { @@ -553,12 +547,10 @@ int kbase_mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, if (likely(!ret)) { mmu_command_instr(kbdev, op_param->kctx_id, mmu_cmd, lock_addr, op_param->mmu_sync_info); -#if MALI_USE_CSF if (flush_op == KBASE_MMU_OP_FLUSH_MEM && kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa && kbdev->pm.backend.l2_state == KBASE_L2_PEND_OFF) ret = wait_l2_power_trans_complete(kbdev); -#endif } return ret; @@ -587,6 +579,7 @@ int kbase_mmu_hw_do_flush_on_gpu_ctrl(struct kbase_device *kbdev, struct kbase_a if (flush_op == KBASE_MMU_OP_FLUSH_PT) gpu_cmd = GPU_COMMAND_CACHE_CLN_INV_L2; + /* 1. Issue MMU_AS_CONTROL.COMMAND.LOCK operation. */ ret = mmu_hw_do_lock(kbdev, as, op_param); if (ret) @@ -598,14 +591,12 @@ int kbase_mmu_hw_do_flush_on_gpu_ctrl(struct kbase_device *kbdev, struct kbase_a /* 3. Issue MMU_AS_CONTROL.COMMAND.UNLOCK operation. */ ret2 = kbase_mmu_hw_do_unlock_no_addr(kbdev, as, op_param); -#if MALI_USE_CSF if (!ret && !ret2) { if (flush_op == KBASE_MMU_OP_FLUSH_MEM && kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa && kbdev->pm.backend.l2_state == KBASE_L2_PEND_OFF) ret = wait_l2_power_trans_complete(kbdev); } -#endif return ret ?: ret2; } @@ -629,20 +620,14 @@ void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, /* Clear the page (and bus fault IRQ as well in case one occurred) */ pf_bf_mask = MMU_PAGE_FAULT(as->number); -#if !MALI_USE_CSF - if (type == KBASE_MMU_FAULT_TYPE_BUS || type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) - pf_bf_mask |= MMU_BUS_ERROR(as->number); -#endif kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_CLEAR), pf_bf_mask); -#if MALI_USE_CSF /* For valid page faults, this function is called just before unblocking the MMU (which * would in turn unblock the MCU firmware) and so this is an opportune location to * update the page fault counter value in firmware visible memory. */ if (likely(type == KBASE_MMU_FAULT_TYPE_PAGE) && kbdev->csf.page_fault_cnt_ptr) *kbdev->csf.page_fault_cnt_ptr = ++kbdev->csf.page_fault_cnt; -#endif unlock: spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); @@ -670,10 +655,6 @@ void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, irq_mask = kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK)) | MMU_PAGE_FAULT(as->number); -#if !MALI_USE_CSF - if (type == KBASE_MMU_FAULT_TYPE_BUS || type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) - irq_mask |= MMU_BUS_ERROR(as->number); -#endif kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK), irq_mask); unlock: diff --git a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_clk_rate_trace.c b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_clk_rate_trace.c index f0995a4cb22e..b3a9331b1c75 100644 --- a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_clk_rate_trace.c +++ b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_clk_rate_trace.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -24,19 +24,15 @@ #include #include "mali_kbase_config_platform.h" -#if MALI_USE_CSF #include -#endif static void *enumerate_gpu_clk(struct kbase_device *kbdev, unsigned int index) { if (index >= kbdev->nr_clocks) return NULL; -#if MALI_USE_CSF if (of_machine_is_compatible("arm,juno")) WARN_ON(kbdev->nr_clocks != 1); -#endif return kbdev->clocks[index]; } @@ -44,7 +40,6 @@ static void *enumerate_gpu_clk(struct kbase_device *kbdev, unsigned int index) static unsigned long get_gpu_clk_rate(struct kbase_device *kbdev, void *gpu_clk_handle) { CSTD_UNUSED(kbdev); -#if MALI_USE_CSF /* On Juno fpga platforms, the GPU clock rate is reported as 600 MHZ at * the boot time. Then after the first call to kbase_devfreq_target() * the clock rate is reported as 450 MHZ and the frequency does not @@ -60,7 +55,6 @@ static unsigned long get_gpu_clk_rate(struct kbase_device *kbdev, void *gpu_clk_ */ if (of_machine_is_compatible("arm,juno")) return arch_timer_get_cntfrq(); -#endif return clk_get_rate((struct clk *)gpu_clk_handle); } @@ -79,11 +73,9 @@ static int gpu_clk_notifier_register(struct kbase_device *kbdev, void *gpu_clk_h sizeof(((struct kbase_gpu_clk_notifier_data *)0)->gpu_clk_handle), "mismatch in the size of clk member"); -#if MALI_USE_CSF /* Frequency is fixed on Juno platforms */ if (of_machine_is_compatible("arm,juno")) return 0; -#endif return clk_notifier_register((struct clk *)gpu_clk_handle, nb); } @@ -93,10 +85,8 @@ static void gpu_clk_notifier_unregister(struct kbase_device *kbdev, void *gpu_cl { CSTD_UNUSED(kbdev); -#if MALI_USE_CSF if (of_machine_is_compatible("arm,juno")) return; -#endif clk_notifier_unregister((struct clk *)gpu_clk_handle, nb); } diff --git a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_config_devicetree.c b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_config_devicetree.c index 7c07f4f96f56..02bcc94d9d61 100644 --- a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_config_devicetree.c +++ b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_config_devicetree.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -42,12 +42,7 @@ void kbase_platform_unregister(void) #endif #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -#else -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]) -#endif { return 1; } diff --git a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_runtime_pm.c b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_runtime_pm.c index b580bfee0078..ef060fabdbaf 100644 --- a/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_runtime_pm.c +++ b/drivers/gpu/arm/valhall/platform/devicetree/mali_kbase_runtime_pm.c @@ -60,7 +60,15 @@ static void disable_gpu_power_control(struct kbase_device *kbdev) ; else if (__clk_is_enabled(kbdev->clocks[i])) { clk_disable_unprepare(kbdev->clocks[i]); - WARN_ON(__clk_is_enabled(kbdev->clocks[i])); + if (__clk_is_enabled(kbdev->clocks[i])) + /* + * Clock won't be disabled when: + * 1. the clock_count for clock[i] is not reaching 0. + * 2. the clocks[i] is fixed-clock. + * Printing logs just for debugging purpose. + */ + dev_dbg(kbdev->dev, "%s Clock %d (%s) is not disabled.\n", __func__, + i, __clk_get_name(kbdev->clocks[i])); } } @@ -77,15 +85,12 @@ static void disable_gpu_power_control(struct kbase_device *kbdev) static int pm_callback_power_on(struct kbase_device *kbdev) { - int ret = 1; /* Assume GPU has been powered off */ - int error; unsigned long flags; dev_dbg(kbdev->dev, "%s %pK\n", __func__, (void *)kbdev->dev->pm_domain); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); WARN_ON(kbase_io_is_gpu_powered(kbdev)); -#if MALI_USE_CSF if (likely(kbdev->csf.firmware_inited)) { WARN_ON(!kbdev->pm.active_count); WARN_ON(kbdev->pm.runtime_active); @@ -93,27 +98,8 @@ static int pm_callback_power_on(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); enable_gpu_power_control(kbdev); - CSTD_UNUSED(error); -#else - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#ifdef KBASE_PM_RUNTIME - error = pm_runtime_get_sync(kbdev->dev); - if (error == 1) { - /* - * Let core know that the chip has not been - * powered off, so we can save on re-initialization. - */ - ret = 0; - } - dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); -#else - enable_gpu_power_control(kbdev); -#endif /* KBASE_PM_RUNTIME */ - -#endif /* MALI_USE_CSF */ - - return ret; + return 1; } static void pm_callback_power_off(struct kbase_device *kbdev) @@ -124,37 +110,26 @@ static void pm_callback_power_off(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); WARN_ON(kbase_io_is_gpu_powered(kbdev)); -#if MALI_USE_CSF if (likely(kbdev->csf.firmware_inited)) { -#ifdef CONFIG_MALI_VALHALL_DEBUG - WARN_ON(kbase_csf_scheduler_get_nr_active_csgs(kbdev)); -#endif + if (IS_ENABLED(CONFIG_MALI_VALHALL_DEBUG)) + WARN_ON(kbase_csf_scheduler_get_nr_active_csgs(kbdev) && + !kbase_io_is_aw_removed(kbdev)); WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_OFF); } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* Power down the GPU immediately */ disable_gpu_power_control(kbdev); -#else /* MALI_USE_CSF */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -#ifdef KBASE_PM_RUNTIME - pm_runtime_mark_last_busy(kbdev->dev); - pm_runtime_put_autosuspend(kbdev->dev); -#else - /* Power down the GPU immediately as runtime PM is disabled */ - disable_gpu_power_control(kbdev); -#endif -#endif /* MALI_USE_CSF */ } -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) static void pm_callback_runtime_gpu_active(struct kbase_device *kbdev) { unsigned long flags; int error; lockdep_assert_held(&kbdev->pm.lock); + if (!IS_ENABLED(CONFIG_PM)) + return; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); WARN_ON(!kbase_io_is_gpu_powered(kbdev)); @@ -179,6 +154,8 @@ static void pm_callback_runtime_gpu_idle(struct kbase_device *kbdev) unsigned long flags; lockdep_assert_held(&kbdev->pm.lock); + if (!IS_ENABLED(CONFIG_PM)) + return; dev_dbg(kbdev->dev, "%s", __func__); @@ -193,14 +170,14 @@ static void pm_callback_runtime_gpu_idle(struct kbase_device *kbdev) pm_runtime_put_autosuspend(kbdev->dev); kbdev->pm.runtime_active = false; } -#endif -#ifdef KBASE_PM_RUNTIME static int kbase_device_runtime_init(struct kbase_device *kbdev) { int ret = 0; dev_dbg(kbdev->dev, "%s\n", __func__); + if (!IS_ENABLED(CONFIG_PM)) + return 0; pm_runtime_set_autosuspend_delay(kbdev->dev, AUTO_SUSPEND_DELAY); pm_runtime_use_autosuspend(kbdev->dev); @@ -211,11 +188,14 @@ static int kbase_device_runtime_init(struct kbase_device *kbdev) if (!pm_runtime_enabled(kbdev->dev)) { dev_warn(kbdev->dev, "pm_runtime not enabled"); ret = -EINVAL; - } else if (atomic_read(&kbdev->dev->power.usage_count)) { + } +#if IS_ENABLED(CONFIG_PM) + else if (atomic_read(&kbdev->dev->power.usage_count)) { dev_warn(kbdev->dev, "%s: Device runtime usage count unexpectedly non zero %d", __func__, atomic_read(&kbdev->dev->power.usage_count)); ret = -EINVAL; } +#endif return ret; } @@ -223,32 +203,26 @@ static int kbase_device_runtime_init(struct kbase_device *kbdev) static void kbase_device_runtime_disable(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); - + if (!IS_ENABLED(CONFIG_PM)) + return; +#if IS_ENABLED(CONFIG_PM) if (atomic_read(&kbdev->dev->power.usage_count)) dev_warn(kbdev->dev, "%s: Device runtime usage count unexpectedly non zero %d", __func__, atomic_read(&kbdev->dev->power.usage_count)); - +#endif pm_runtime_disable(kbdev->dev); } -#endif /* KBASE_PM_RUNTIME */ static int pm_callback_runtime_on(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); -#if !MALI_USE_CSF - enable_gpu_power_control(kbdev); -#endif return 0; } static void pm_callback_runtime_off(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); - -#if !MALI_USE_CSF - disable_gpu_power_control(kbdev); -#endif } static void pm_callback_resume(struct kbase_device *kbdev) @@ -269,23 +243,10 @@ struct kbase_pm_callback_conf pm_callbacks = { .power_off_callback = pm_callback_power_off, .power_suspend_callback = pm_callback_suspend, .power_resume_callback = pm_callback_resume, -#ifdef KBASE_PM_RUNTIME .power_runtime_init_callback = kbase_device_runtime_init, .power_runtime_term_callback = kbase_device_runtime_disable, .power_runtime_on_callback = pm_callback_runtime_on, .power_runtime_off_callback = pm_callback_runtime_off, -#else /* KBASE_PM_RUNTIME */ - .power_runtime_init_callback = NULL, - .power_runtime_term_callback = NULL, - .power_runtime_on_callback = NULL, - .power_runtime_off_callback = NULL, -#endif /* KBASE_PM_RUNTIME */ - -#if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) .power_runtime_gpu_idle_callback = pm_callback_runtime_gpu_idle, .power_runtime_gpu_active_callback = pm_callback_runtime_gpu_active, -#else - .power_runtime_gpu_idle_callback = NULL, - .power_runtime_gpu_active_callback = NULL, -#endif }; diff --git a/drivers/gpu/arm/valhall/platform/meson/mali_kbase_config_meson.c b/drivers/gpu/arm/valhall/platform/meson/mali_kbase_config_meson.c index 7c07f4f96f56..02bcc94d9d61 100644 --- a/drivers/gpu/arm/valhall/platform/meson/mali_kbase_config_meson.c +++ b/drivers/gpu/arm/valhall/platform/meson/mali_kbase_config_meson.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -42,12 +42,7 @@ void kbase_platform_unregister(void) #endif #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -#else -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]) -#endif { return 1; } diff --git a/drivers/gpu/arm/valhall/platform/meson/mali_kbase_runtime_pm.c b/drivers/gpu/arm/valhall/platform/meson/mali_kbase_runtime_pm.c index bd3b4b5e2aa3..e51e251c8380 100644 --- a/drivers/gpu/arm/valhall/platform/meson/mali_kbase_runtime_pm.c +++ b/drivers/gpu/arm/valhall/platform/meson/mali_kbase_runtime_pm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -126,7 +126,15 @@ static void disable_gpu_power_control(struct kbase_device *kbdev) ; else if (__clk_is_enabled(kbdev->clocks[i])) { clk_disable_unprepare(kbdev->clocks[i]); - WARN_ON(__clk_is_enabled(kbdev->clocks[i])); + if (__clk_is_enabled(kbdev->clocks[i])) + /* + * Clock won't be disabled when: + * 1. the clock_count for clock[i] is not reaching 0. + * 2. the clocks[i] is fixed-clock. + * Printing logs just for debugging purpose. + */ + dev_dbg(kbdev->dev, "%s Clock %d (%s) is not disabled.\n", __func__, + i, __clk_get_name(kbdev->clocks[i])); } } @@ -147,19 +155,18 @@ static int pm_callback_power_on(struct kbase_device *kbdev) dev_dbg(kbdev->dev, "%s %pK\n", __func__, (void *)kbdev->dev->pm_domain); -#ifdef KBASE_PM_RUNTIME - error = pm_runtime_get_sync(kbdev->dev); - if (error == 1) { - /* - * Let core know that the chip has not been - * powered off, so we can save on re-initialization. - */ - ret = 0; - } - dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); -#else - enable_gpu_power_control(kbdev); -#endif + if (IS_ENABLED(CONFIG_PM)) { + error = pm_runtime_get_sync(kbdev->dev); + if (error == 1) { + /* + * Let core know that the chip has not been + * powered off, so we can save on re-initialization. + */ + ret = 0; + } + dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); + } else + enable_gpu_power_control(kbdev); return ret; } @@ -168,21 +175,22 @@ static void pm_callback_power_off(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); -#ifdef KBASE_PM_RUNTIME - pm_runtime_mark_last_busy(kbdev->dev); - pm_runtime_put_autosuspend(kbdev->dev); -#else - /* Power down the GPU immediately as runtime PM is disabled */ - disable_gpu_power_control(kbdev); -#endif + if (IS_ENABLED(CONFIG_PM)) { + pm_runtime_mark_last_busy(kbdev->dev); + pm_runtime_put_autosuspend(kbdev->dev); + } else { + /* Power down the GPU immediately as runtime PM is disabled */ + disable_gpu_power_control(kbdev); + } } -#ifdef KBASE_PM_RUNTIME static int kbase_device_runtime_init(struct kbase_device *kbdev) { int ret = 0; dev_dbg(kbdev->dev, "%s\n", __func__); + if (!IS_ENABLED(CONFIG_PM)) + return 0; pm_runtime_set_autosuspend_delay(kbdev->dev, AUTO_SUSPEND_DELAY); pm_runtime_use_autosuspend(kbdev->dev); @@ -193,11 +201,14 @@ static int kbase_device_runtime_init(struct kbase_device *kbdev) if (!pm_runtime_enabled(kbdev->dev)) { dev_warn(kbdev->dev, "pm_runtime not enabled"); ret = -EINVAL; - } else if (atomic_read(&kbdev->dev->power.usage_count)) { + } +#if IS_ENABLED(CONFIG_PM) + else if (atomic_read(&kbdev->dev->power.usage_count)) { dev_warn(kbdev->dev, "%s: Device runtime usage count unexpectedly non zero %d", __func__, atomic_read(&kbdev->dev->power.usage_count)); ret = -EINVAL; } +#endif /* allocate resources for reset */ if (!ret) @@ -209,28 +220,32 @@ static int kbase_device_runtime_init(struct kbase_device *kbdev) static void kbase_device_runtime_disable(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); + if (!IS_ENABLED(CONFIG_PM)) + return; +#if IS_ENABLED(CONFIG_PM) if (atomic_read(&kbdev->dev->power.usage_count)) dev_warn(kbdev->dev, "%s: Device runtime usage count unexpectedly non zero %d", __func__, atomic_read(&kbdev->dev->power.usage_count)); +#endif pm_runtime_disable(kbdev->dev); } -#endif /* KBASE_PM_RUNTIME */ static int pm_callback_runtime_on(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); + if (IS_ENABLED(CONFIG_PM)) + enable_gpu_power_control(kbdev); - enable_gpu_power_control(kbdev); return 0; } static void pm_callback_runtime_off(struct kbase_device *kbdev) { dev_dbg(kbdev->dev, "%s\n", __func__); - - disable_gpu_power_control(kbdev); + if (IS_ENABLED(CONFIG_PM)) + disable_gpu_power_control(kbdev); } static void pm_callback_resume(struct kbase_device *kbdev) @@ -251,15 +266,8 @@ struct kbase_pm_callback_conf pm_callbacks = { .power_suspend_callback = pm_callback_suspend, .power_resume_callback = pm_callback_resume, .soft_reset_callback = pm_callback_soft_reset, -#ifdef KBASE_PM_RUNTIME .power_runtime_init_callback = kbase_device_runtime_init, .power_runtime_term_callback = kbase_device_runtime_disable, .power_runtime_on_callback = pm_callback_runtime_on, .power_runtime_off_callback = pm_callback_runtime_off, -#else /* KBASE_PM_RUNTIME */ - .power_runtime_init_callback = NULL, - .power_runtime_term_callback = NULL, - .power_runtime_on_callback = NULL, - .power_runtime_off_callback = NULL, -#endif /* KBASE_PM_RUNTIME */ }; diff --git a/drivers/gpu/arm/valhall/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/valhall/platform/rk/mali_kbase_config_rk.c index f87a6f50503a..628be3a40c86 100755 --- a/drivers/gpu/arm/valhall/platform/rk/mali_kbase_config_rk.c +++ b/drivers/gpu/arm/valhall/platform/rk/mali_kbase_config_rk.c @@ -18,9 +18,7 @@ #include #include -#if MALI_USE_CSF #include -#endif #include #include diff --git a/drivers/gpu/arm/valhall/platform/vexpress/mali_kbase_config_vexpress.c b/drivers/gpu/arm/valhall/platform/vexpress/mali_kbase_config_vexpress.c index c655aa62b526..e1bfe2f26f77 100644 --- a/drivers/gpu/arm/valhall/platform/vexpress/mali_kbase_config_vexpress.c +++ b/drivers/gpu/arm/valhall/platform/vexpress/mali_kbase_config_vexpress.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -63,12 +63,7 @@ struct kbase_platform_config *kbase_get_platform_config(void) } #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -#else -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]) -#endif { return 1; } diff --git a/drivers/gpu/arm/valhall/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/drivers/gpu/arm/valhall/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c index d8a566d9d3c0..45ddda005d11 100644 --- a/drivers/gpu/arm/valhall/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c +++ b/drivers/gpu/arm/valhall/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -62,12 +62,7 @@ struct kbase_platform_config *kbase_get_platform_config(void) } #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -#else -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]) -#endif { return 1; } diff --git a/drivers/gpu/arm/valhall/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/drivers/gpu/arm/valhall/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c index 63e990cdf6cf..3285277ccfb0 100644 --- a/drivers/gpu/arm/valhall/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c +++ b/drivers/gpu/arm/valhall/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -62,12 +62,7 @@ struct kbase_platform_config *kbase_get_platform_config(void) } #ifdef CONFIG_MALI_VALHALL_DVFS -#if MALI_USE_CSF int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -#else -int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, - u32 util_cl_share[2]) -#endif { return 1; } diff --git a/drivers/gpu/arm/valhall/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c b/drivers/gpu/arm/valhall/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c index 0598d4397e2a..1939e3f3de48 100644 --- a/drivers/gpu/arm/valhall/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c +++ b/drivers/gpu/arm/valhall/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c @@ -176,9 +176,6 @@ static void kutf_set_pm_ctx_active(struct kutf_context *context) kbase_pm_context_active(data->kbdev); kbase_pm_wait_for_desired_state(data->kbdev); -#if !MALI_USE_CSF - kbase_pm_request_gpu_cycle_counter(data->kbdev); -#endif } static void kutf_set_pm_ctx_idle(struct kutf_context *context) @@ -187,9 +184,6 @@ static void kutf_set_pm_ctx_idle(struct kutf_context *context) if (WARN_ON(data->pm_ctx_cnt > 0)) return; -#if !MALI_USE_CSF - kbase_pm_release_gpu_cycle_counter(data->kbdev); -#endif kbase_pm_context_idle(data->kbdev); } diff --git a/drivers/gpu/arm/valhall/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/drivers/gpu/arm/valhall/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c index 6a143bb01616..4f7d781c0589 100644 --- a/drivers/gpu/arm/valhall/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +++ b/drivers/gpu/arm/valhall/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c @@ -85,7 +85,6 @@ static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) u64 tval; bool has_test_irq = val & test_irq; -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { status_reg_enum = HOST_POWER_ENUM(PWR_IRQ_STATUS); clear_reg_enum = HOST_POWER_ENUM(PWR_IRQ_CLEAR); @@ -93,7 +92,6 @@ static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) val = kbase_reg_read32(kbdev, status_reg_enum); has_test_irq = val & test_irq; } -#endif if (has_test_irq) { tval = ktime_get_real_ns(); @@ -201,12 +199,10 @@ static void mali_kutf_irq_latency(struct kutf_context *context) triggered = false; /* Trigger fake IRQ */ -#if MALI_USE_CSF if (kbdev->pm.backend.has_host_pwr_iface) { reg_enum = HOST_POWER_ENUM(PWR_IRQ_RAWSTAT); test_irq = PWR_IRQ_POWER_CHANGED_SINGLE; } -#endif kbase_reg_write32(kbdev, reg_enum, test_irq); if (wait_event_timeout(wait, triggered, IRQ_TIMEOUT) == 0) { diff --git a/drivers/gpu/arm/valhall/thirdparty/mali_kbase_mmap.c b/drivers/gpu/arm/valhall/thirdparty/mali_kbase_mmap.c index 162f82e5493a..8a71ed685de4 100644 --- a/drivers/gpu/arm/valhall/thirdparty/mali_kbase_mmap.c +++ b/drivers/gpu/arm/valhall/thirdparty/mali_kbase_mmap.c @@ -487,21 +487,6 @@ unsigned long kbase_context_get_unmapped_area(struct kbase_context *const kctx, align_mask = align_offset - 1; is_shader_code = true; } -#if !MALI_USE_CSF - } else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { - unsigned long extension_bytes = - (unsigned long)(reg->extension << PAGE_SHIFT); - /* kbase_check_alloc_sizes() already satisfies - * these checks, but they're here to avoid - * maintenance hazards due to the assumptions - * involved - */ - WARN_ON(reg->extension > (ULONG_MAX >> PAGE_SHIFT)); - WARN_ON(reg->initial_commit > (ULONG_MAX >> PAGE_SHIFT)); - WARN_ON(!is_power_of_2(extension_bytes)); - align_mask = extension_bytes - 1; - align_offset = extension_bytes - (reg->initial_commit << PAGE_SHIFT); -#endif /* !MALI_USE_CSF */ } else if (reg->flags & KBASE_REG_GPU_VA_SAME_4GB_PAGE) { is_same_4gb_page = true; } diff --git a/drivers/gpu/arm/valhall/tl/Kbuild b/drivers/gpu/arm/valhall/tl/Kbuild index 35c3fabfc609..3dce47d3c0de 100644 --- a/drivers/gpu/arm/valhall/tl/Kbuild +++ b/drivers/gpu/arm/valhall/tl/Kbuild @@ -22,11 +22,6 @@ valhall_kbase-y += \ tl/mali_kbase_timeline.o \ tl/mali_kbase_timeline_io.o \ tl/mali_kbase_tlstream.o \ - tl/mali_kbase_tracepoints.o + tl/mali_kbase_tracepoints.o \ + tl/backend/mali_kbase_timeline_csf.o - -ifeq ($(CONFIG_MALI_VALHALL_CSF_SUPPORT),y) - valhall_kbase-y += tl/backend/mali_kbase_timeline_csf.o -else - valhall_kbase-y += tl/backend/mali_kbase_timeline_jm.o -endif diff --git a/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_csf.c b/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_csf.c index 8c65b62ae46d..60880d1cff13 100644 --- a/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_csf.c +++ b/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_csf.c @@ -33,12 +33,10 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) struct kbase_timeline *timeline = kbdev->timeline; struct kbase_tlstream *summary = &kbdev->timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]; u32 const num_sb_entries = kbdev->gpu_props.gpu_id.arch_major >= 11 ? 16 : 8; - u32 const supports_gpu_sleep = -#ifdef KBASE_PM_RUNTIME - test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); -#else - false; -#endif /* KBASE_PM_RUNTIME */ + u32 const supports_gpu_sleep = IS_ENABLED(CONFIG_PM) ? + test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, + &kbdev->pm.backend.gpu_sleep_allowed) : + false; /* Summarize the Address Space objects. */ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) diff --git a/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_jm.c b/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_jm.c deleted file mode 100644 index 3e9e6e864125..000000000000 --- a/drivers/gpu/arm/valhall/tl/backend/mali_kbase_timeline_jm.c +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -/* - * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#include -#include -#include - -#include - -void kbase_create_timeline_objects(struct kbase_device *kbdev) -{ - unsigned int lpu_id; - int as_nr; - struct kbase_context *kctx; - struct kbase_timeline *timeline = kbdev->timeline; - struct kbase_tlstream *summary = &timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]; - - /* Summarize the LPU objects. */ - for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { - void *lpu = &kbdev->gpu_props.js_features[lpu_id]; - - __kbase_tlstream_tl_new_lpu(summary, lpu, lpu_id, 0); - } - - /* Summarize the Address Space objects. */ - for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) - __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], (u32)as_nr); - - /* Create GPU object and make it retain all LPUs and address spaces. */ - __kbase_tlstream_tl_new_gpu(summary, kbdev, kbdev->id, kbdev->gpu_props.num_cores); - - for (lpu_id = 0; lpu_id < kbdev->gpu_props.num_job_slots; lpu_id++) { - void *lpu = &kbdev->gpu_props.js_features[lpu_id]; - __kbase_tlstream_tl_lifelink_lpu_gpu(summary, lpu, kbdev); - } - - for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) - __kbase_tlstream_tl_lifelink_as_gpu(summary, &kbdev->as[as_nr], kbdev); - - /* Lock the context list, to ensure no changes to the list are made - * while we're summarizing the contexts and their contents. - */ - mutex_lock(&timeline->tl_kctx_list_lock); - - /* For each context in the device... */ - list_for_each_entry(kctx, &timeline->tl_kctx_list, tl_kctx_list_node) { - /* Summarize the context itself */ - __kbase_tlstream_tl_new_ctx(summary, kctx, kctx->id, (u32)(kctx->tgid)); - } - - /* Reset body stream buffers while holding the kctx lock. - * This ensures we can't fire both summary and normal tracepoints for - * the same objects. - * If we weren't holding the lock, it's possible that the summarized - * objects could have been created, destroyed, or used after we - * constructed the summary stream tracepoints, but before we reset - * the body stream, resulting in losing those object event tracepoints. - */ - kbase_timeline_streams_body_reset(timeline); - - mutex_unlock(&timeline->tl_kctx_list_lock); - - /* Static object are placed into summary packet that needs to be - * transmitted first. Flush all streams to make it available to - * user space. - */ - kbase_timeline_streams_flush(timeline); -} diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline.c b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline.c index 96f780945a4b..22de4c8a7e9a 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline.c +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline.c @@ -94,9 +94,8 @@ static void kbasep_timeline_autoflush_timer_callback(struct timer_list *timer) int kbase_timeline_init(struct kbase_timeline **timeline, atomic_t *timeline_flags) { struct kbase_timeline *result; -#if MALI_USE_CSF struct kbase_tlstream *csffw_stream; -#endif + int i; if (!timeline || !timeline_flags) return -EINVAL; @@ -109,6 +108,9 @@ int kbase_timeline_init(struct kbase_timeline **timeline, atomic_t *timeline_fla mutex_init(&result->streams_buf_lock); init_waitqueue_head(&result->event_queue); + for (i = 0; i < TL_STREAM_TYPE_COUNT; i++) + spin_lock_init(&result->streams[i].lock); + /* Initialize the kctx list */ mutex_init(&result->tl_kctx_list_lock); INIT_LIST_HEAD(&result->tl_kctx_list); @@ -118,10 +120,8 @@ int kbase_timeline_init(struct kbase_timeline **timeline, atomic_t *timeline_fla kbase_timer_setup(&result->autoflush_timer, kbasep_timeline_autoflush_timer_callback); result->timeline_flags = timeline_flags; -#if MALI_USE_CSF csffw_stream = &result->streams[TL_STREAM_TYPE_CSFFW]; kbase_csf_tl_reader_init(&result->csf_tl_reader, csffw_stream); -#endif *timeline = result; return 0; @@ -134,9 +134,7 @@ void kbase_timeline_term(struct kbase_timeline *timeline) if (!timeline) return; -#if MALI_USE_CSF kbase_csf_tl_reader_term(&timeline->csf_tl_reader); -#endif WARN_ON(!list_empty(&timeline->tl_kctx_list)); @@ -208,7 +206,6 @@ int kbase_timeline_acquire(struct kbase_device *kbdev, u32 flags) if (atomic_cmpxchg(timeline->timeline_flags, 0, (int)timeline_flags)) return -EBUSY; -#if MALI_USE_CSF if (flags & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) { err = kbase_csf_tl_reader_start(&timeline->csf_tl_reader, kbdev); if (err) { @@ -216,7 +213,6 @@ int kbase_timeline_acquire(struct kbase_device *kbdev, u32 flags) return err; } } -#endif /* Reset and initialize header streams. */ kbase_tlstream_reset(&timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]); @@ -224,18 +220,6 @@ int kbase_timeline_acquire(struct kbase_device *kbdev, u32 flags) timeline->obj_header_btc = obj_desc_header_size; timeline->aux_header_btc = aux_desc_header_size; -#if !MALI_USE_CSF - /* If job dumping is enabled, readjust the software event's - * timeout as the default value of 3 seconds is often - * insufficient. - */ - if (flags & BASE_TLSTREAM_JOB_DUMPING_ENABLED) { - dev_info(kbdev->dev, - "Job dumping is enabled, readjusting the software event's timeout\n"); - atomic_set(&kbdev->js_data.soft_job_timeout_ms, 1800000); - } -#endif /* !MALI_USE_CSF */ - /* Summary stream was cleared during acquire. * Create static timeline objects that will be * read by client. @@ -285,9 +269,7 @@ void kbase_timeline_release(struct kbase_timeline *timeline) if (time_to_sleep > 0) msleep_interruptible(time_to_sleep); -#if MALI_USE_CSF kbase_csf_tl_reader_stop(&timeline->csf_tl_reader); -#endif /* Stop autoflush timer before releasing access to streams. */ atomic_set(&timeline->autoflush_timer_active, 0); @@ -305,14 +287,12 @@ int kbase_timeline_streams_flush(struct kbase_timeline *timeline) if (WARN_ON(!timeline)) return -EINVAL; -#if MALI_USE_CSF { int ret = kbase_csf_tl_reader_flush_buffer(&timeline->csf_tl_reader); if (ret > 0) has_bytes = true; } -#endif for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) { nbytes = kbase_tlstream_flush_stream(&timeline->streams[stype]); @@ -326,9 +306,7 @@ void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline) { kbase_tlstream_reset(&timeline->streams[TL_STREAM_TYPE_OBJ]); kbase_tlstream_reset(&timeline->streams[TL_STREAM_TYPE_AUX]); -#if MALI_USE_CSF kbase_tlstream_reset(&timeline->streams[TL_STREAM_TYPE_CSFFW]); -#endif } void kbase_timeline_pre_kbase_context_destroy(struct kbase_context *kctx) @@ -370,9 +348,7 @@ void kbase_timeline_post_kbase_context_create(struct kbase_context *kctx) * never in parallel with it. If fired in parallel, we could get * duplicate creation tracepoints. */ -#if MALI_USE_CSF KBASE_TLSTREAM_TL_KBASE_NEW_CTX(kbdev, kctx->id, kbdev->id); -#endif /* Trace with the AOM tracepoint even in CSF for dumping */ KBASE_TLSTREAM_TL_NEW_CTX(kbdev, kctx, kctx->id, 0); @@ -385,9 +361,7 @@ void kbase_timeline_post_kbase_context_destroy(struct kbase_context *kctx) /* Trace with the AOM tracepoint even in CSF for dumping */ KBASE_TLSTREAM_TL_DEL_CTX(kbdev, kctx); -#if MALI_USE_CSF KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kbdev, kctx->id); -#endif /* Flush the timeline stream, so the user can see the termination * tracepoints being fired. diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_io.c b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_io.c index 522fd9e0964a..e3b4b0da17ce 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_io.c +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_io.c @@ -146,11 +146,8 @@ static int kbasep_timeline_io_packet_pending(struct kbase_timeline *timeline, */ static int kbasep_timeline_has_header_data(struct kbase_timeline *timeline) { - return timeline->obj_header_btc || timeline->aux_header_btc -#if MALI_USE_CSF - || timeline->csf_tl_reader.tl_header.btc -#endif - ; + return timeline->obj_header_btc || timeline->aux_header_btc || + timeline->csf_tl_reader.tl_header.btc; } /** @@ -217,12 +214,10 @@ static inline int kbasep_timeline_copy_headers(struct kbase_timeline *timeline, if (copy_stream_header(buffer, size, copy_len, aux_desc_header, aux_desc_header_size, &timeline->aux_header_btc)) return -1; -#if MALI_USE_CSF if (copy_stream_header(buffer, size, copy_len, timeline->csf_tl_reader.tl_header.data, timeline->csf_tl_reader.tl_header.size, &timeline->csf_tl_reader.tl_header.btc)) return -1; -#endif return 0; } diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_priv.h b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_priv.h index 59848ba7bf0f..ddcdc5816ca2 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_priv.h +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_timeline_priv.h @@ -25,10 +25,8 @@ #include #include "mali_kbase_tlstream.h" -#if MALI_USE_CSF #include "csf/mali_kbase_csf_tl_reader.h" #include "csf/mali_kbase_csf_trace_buffer.h" -#endif #include #include @@ -74,9 +72,7 @@ struct kbase_timeline { size_t obj_header_btc; size_t aux_header_btc; ktime_t last_acquire_time; -#if MALI_USE_CSF struct kbase_csf_tl_reader csf_tl_reader; -#endif }; void kbase_create_timeline_objects(struct kbase_device *kbdev); diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.c b/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.c index 117417c30183..1f504e253140 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.c +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2015-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -120,14 +120,12 @@ static const struct { TL_PACKET_TYPE_BODY, TL_STREAM_ID_KERNEL, }, -#if MALI_USE_CSF { TL_PACKET_FAMILY_TL, TL_PACKET_CLASS_OBJ, TL_PACKET_TYPE_BODY, TL_STREAM_ID_CSFFW, }, -#endif }; void kbase_tlstream_init(struct kbase_tlstream *stream, enum tl_stream_type stream_type, @@ -138,8 +136,6 @@ void kbase_tlstream_init(struct kbase_tlstream *stream, enum tl_stream_type stre KBASE_DEBUG_ASSERT(stream); KBASE_DEBUG_ASSERT(stream_type < TL_STREAM_TYPE_COUNT); - spin_lock_init(&stream->lock); - /* All packets carrying tracepoints shall be numbered. */ if (tl_stream_cfg[stream_type].pkt_type == TL_PACKET_TYPE_BODY) stream->numbered = 1; @@ -219,6 +215,7 @@ char *kbase_tlstream_msgbuf_acquire(struct kbase_tlstream *stream, size_t msg_si size_t wb_size; KBASE_DEBUG_ASSERT(PACKET_SIZE - PACKET_HEADER_SIZE - PACKET_NUMBER_SIZE >= msg_size); + WARN_ON(!stream->buffer); spin_lock_irqsave(&stream->lock, *flags); @@ -259,6 +256,9 @@ size_t kbase_tlstream_flush_stream(struct kbase_tlstream *stream) size_t wb_size; size_t min_size = PACKET_HEADER_SIZE; + if (!stream->buffer) + return 0; + if (stream->numbered) min_size += PACKET_NUMBER_SIZE; diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.h b/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.h index d5745796e8e2..a74f5c044db4 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.h +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_tlstream.h @@ -99,9 +99,7 @@ enum tl_stream_type { TL_STREAM_TYPE_OBJ_SUMMARY = TL_STREAM_TYPE_FIRST, TL_STREAM_TYPE_OBJ, TL_STREAM_TYPE_AUX, -#if MALI_USE_CSF TL_STREAM_TYPE_CSFFW, -#endif TL_STREAM_TYPE_COUNT }; diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.c b/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.c index 2cc556583d8f..a8052716dee8 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.c +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -83,7 +83,7 @@ enum tl_msg_id_obj { KBASE_TL_ATTRIB_ATOM_PRIORITY, KBASE_TL_ATTRIB_ATOM_STATE, KBASE_TL_ATTRIB_ATOM_PRIORITIZED, - KBASE_TL_ATTRIB_ATOM_JIT, + KBASE_JD_AS_INFO, KBASE_TL_KBASE_NEW_DEVICE, KBASE_TL_KBASE_GPUCMDQUEUE_KICK, KBASE_TL_KBASE_DEVICE_PROGRAM_CSG, @@ -354,10 +354,10 @@ enum tl_msg_id_obj { "atom caused priority change", \ "@p", \ "atom") \ - TRACEPOINT_DESC(KBASE_TL_ATTRIB_ATOM_JIT, \ - "jit done for atom", \ - "@pLLILILLL", \ - "atom,edit_addr,new_addr,jit_flags,mem_flags,j_id,com_pgs,extent,va_pgs") \ + TRACEPOINT_DESC(KBASE_JD_AS_INFO, \ + "address space attributes", \ + "@ILLL", \ + "as_nr,transtab,memattr,transcfg") \ TRACEPOINT_DESC(KBASE_TL_KBASE_NEW_DEVICE, \ "New KBase Device", \ "@IIIIIIII", \ @@ -2043,30 +2043,20 @@ void __kbase_tlstream_tl_attrib_atom_prioritized( kbase_tlstream_msgbuf_release(stream, acq_flags); } -void __kbase_tlstream_tl_attrib_atom_jit( +void __kbase_tlstream_jd_as_info( struct kbase_tlstream *stream, - const void *atom, - u64 edit_addr, - u64 new_addr, - u32 jit_flags, - u64 mem_flags, - u32 j_id, - u64 com_pgs, - u64 extent, - u64 va_pgs + u32 as_nr, + u64 transtab, + u64 memattr, + u64 transcfg ) { - const u32 msg_id = KBASE_TL_ATTRIB_ATOM_JIT; + const u32 msg_id = KBASE_JD_AS_INFO; const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(atom) - + sizeof(edit_addr) - + sizeof(new_addr) - + sizeof(jit_flags) - + sizeof(mem_flags) - + sizeof(j_id) - + sizeof(com_pgs) - + sizeof(extent) - + sizeof(va_pgs) + + sizeof(as_nr) + + sizeof(transtab) + + sizeof(memattr) + + sizeof(transcfg) ; char *buffer; unsigned long acq_flags; @@ -2077,23 +2067,13 @@ void __kbase_tlstream_tl_attrib_atom_jit( pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); pos = kbasep_serialize_timestamp(buffer, pos); pos = kbasep_serialize_bytes(buffer, - pos, &atom, sizeof(atom)); + pos, &as_nr, sizeof(as_nr)); pos = kbasep_serialize_bytes(buffer, - pos, &edit_addr, sizeof(edit_addr)); + pos, &transtab, sizeof(transtab)); pos = kbasep_serialize_bytes(buffer, - pos, &new_addr, sizeof(new_addr)); + pos, &memattr, sizeof(memattr)); pos = kbasep_serialize_bytes(buffer, - pos, &jit_flags, sizeof(jit_flags)); - pos = kbasep_serialize_bytes(buffer, - pos, &mem_flags, sizeof(mem_flags)); - pos = kbasep_serialize_bytes(buffer, - pos, &j_id, sizeof(j_id)); - pos = kbasep_serialize_bytes(buffer, - pos, &com_pgs, sizeof(com_pgs)); - pos = kbasep_serialize_bytes(buffer, - pos, &extent, sizeof(extent)); - pos = kbasep_serialize_bytes(buffer, - pos, &va_pgs, sizeof(va_pgs)); + pos, &transcfg, sizeof(transcfg)); kbase_tlstream_msgbuf_release(stream, acq_flags); } diff --git a/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.h b/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.h index 2763866758c0..09678688805f 100644 --- a/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.h +++ b/drivers/gpu/arm/valhall/tl/mali_kbase_tracepoints.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -382,17 +382,12 @@ void __kbase_tlstream_tl_attrib_atom_prioritized( const void *atom ); -void __kbase_tlstream_tl_attrib_atom_jit( +void __kbase_tlstream_jd_as_info( struct kbase_tlstream *stream, - const void *atom, - u64 edit_addr, - u64 new_addr, - u32 jit_flags, - u64 mem_flags, - u32 j_id, - u64 com_pgs, - u64 extent, - u64 va_pgs + u32 as_nr, + u64 transtab, + u64 memattr, + u64 transcfg ); void __kbase_tlstream_tl_kbase_new_device( @@ -1961,45 +1956,30 @@ struct kbase_tlstream; } while (0) /** - * KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT - jit done for atom + * KBASE_TLSTREAM_JD_AS_INFO - address space attributes * * @kbdev: Kbase device - * @atom: Atom identifier - * @edit_addr: Address edited by jit - * @new_addr: Address placed into the edited location - * @jit_flags: Flags specifying the special requirements for the JIT allocation. - * @mem_flags: Flags defining the properties of a memory region - * @j_id: Unique ID provided by the caller, this is used to pair allocation and free requests. - * @com_pgs: The minimum number of physical pages which should back the allocation. - * @extent: Granularity of physical pages to grow the allocation by during a fault. - * @va_pgs: The minimum number of virtual pages required + * @as_nr: Address space number + * @transtab: Configuration of the TRANSTAB register + * @memattr: Configuration of the MEMATTR register + * @transcfg: Configuration of the TRANSCFG register (or zero if not present) */ -#define KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( \ +#define KBASE_TLSTREAM_JD_AS_INFO( \ kbdev, \ - atom, \ - edit_addr, \ - new_addr, \ - jit_flags, \ - mem_flags, \ - j_id, \ - com_pgs, \ - extent, \ - va_pgs \ + as_nr, \ + transtab, \ + memattr, \ + transcfg \ ) \ do { \ u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ - __kbase_tlstream_tl_attrib_atom_jit( \ + __kbase_tlstream_jd_as_info( \ __TL_DISPATCH_STREAM(kbdev, obj), \ - atom, \ - edit_addr, \ - new_addr, \ - jit_flags, \ - mem_flags, \ - j_id, \ - com_pgs, \ - extent, \ - va_pgs \ + as_nr, \ + transtab, \ + memattr, \ + transcfg \ ); \ } while (0) @@ -2016,7 +1996,6 @@ struct kbase_tlstream; * @kbase_device_supports_gpu_sleep: Whether GPU sleep is supported * @kbase_device_has_neural_engine: Whether neural engine is supported */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ kbdev, \ kbase_device_id, \ @@ -2043,20 +2022,6 @@ struct kbase_tlstream; kbase_device_has_neural_engine \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ - kbdev, \ - kbase_device_id, \ - kbase_device_gpu_core_count, \ - kbase_device_max_num_csgs, \ - kbase_device_as_count, \ - kbase_device_sb_entry_count, \ - kbase_device_has_cross_stream_sync, \ - kbase_device_supports_gpu_sleep, \ - kbase_device_has_neural_engine \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_GPUCMDQUEUE_KICK - Kernel receives a request to process new GPU queue instructions @@ -2065,7 +2030,6 @@ struct kbase_tlstream; * @kernel_ctx_id: Unique ID for the KBase Context * @buffer_gpu_addr: Address of the GPU queue's command buffer */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_GPUCMDQUEUE_KICK( \ kbdev, \ kernel_ctx_id, \ @@ -2080,14 +2044,6 @@ struct kbase_tlstream; buffer_gpu_addr \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_GPUCMDQUEUE_KICK( \ - kbdev, \ - kernel_ctx_id, \ - buffer_gpu_addr \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG - CSG is programmed to a slot @@ -2099,7 +2055,6 @@ struct kbase_tlstream; * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed * @kbase_device_csg_slot_resuming: Whether the csg is being resumed */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ kbdev, \ kbase_device_id, \ @@ -2120,17 +2075,6 @@ struct kbase_tlstream; kbase_device_csg_slot_resuming \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ - kbdev, \ - kbase_device_id, \ - kernel_ctx_id, \ - gpu_cmdq_grp_handle, \ - kbase_device_csg_slot_index, \ - kbase_device_csg_slot_resuming \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG - CSG is deprogrammed from a slot @@ -2139,7 +2083,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @kbase_device_csg_slot_index: The index of the slot in the scheduler whose CSG is being deprogrammed */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ kbdev, \ kbase_device_id, \ @@ -2154,14 +2097,6 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_PROTM_ENTER_CSG - CSG slot is entering protected mode @@ -2170,7 +2105,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @kbase_device_csg_slot_index: The index of the slot in the scheduler whose CSG has entered PMODE */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROTM_ENTER_CSG( \ kbdev, \ kbase_device_id, \ @@ -2185,14 +2119,6 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROTM_ENTER_CSG( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_HALTING_CSG - CSG is halting @@ -2202,7 +2128,6 @@ struct kbase_tlstream; * @kbase_device_csg_slot_index: The index of the slot in the scheduler whose CSG is being halted * @kbase_device_csg_slot_suspending: Whether the csg is being suspended */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_HALTING_CSG( \ kbdev, \ kbase_device_id, \ @@ -2219,15 +2144,6 @@ struct kbase_tlstream; kbase_device_csg_slot_suspending \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_HALTING_CSG( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index, \ - kbase_device_csg_slot_suspending \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_SUSPEND_CSG - CSG is suspended @@ -2236,7 +2152,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @kbase_device_csg_slot_index: The index of the slot in the scheduler whose CSG is being suspended */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_SUSPEND_CSG( \ kbdev, \ kbase_device_id, \ @@ -2251,14 +2166,6 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_SUSPEND_CSG( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_CSG_IDLE - KBase device is notified that CSG is idle. @@ -2267,7 +2174,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @kbase_device_csg_slot_index: The index of the slot in the scheduler whose CSG for which we are receiving an idle notification */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_CSG_IDLE( \ kbdev, \ kbase_device_id, \ @@ -2282,14 +2188,6 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_CSG_IDLE( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_NEW_CTX - New KBase Context @@ -2298,7 +2196,6 @@ struct kbase_tlstream; * @kernel_ctx_id: Unique ID for the KBase Context * @kbase_device_id: The ID of the physical hardware */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ kbdev, \ kernel_ctx_id, \ @@ -2313,14 +2210,6 @@ struct kbase_tlstream; kbase_device_id \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ - kbdev, \ - kernel_ctx_id, \ - kbase_device_id \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEL_CTX - Delete KBase Context @@ -2328,7 +2217,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kernel_ctx_id: Unique ID for the KBase Context */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ kbdev, \ kernel_ctx_id \ @@ -2341,13 +2229,6 @@ struct kbase_tlstream; kernel_ctx_id \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ - kbdev, \ - kernel_ctx_id \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE - New KCPU Queue @@ -2358,7 +2239,6 @@ struct kbase_tlstream; * @kernel_ctx_id: Unique ID for the KBase Context * @kcpuq_num_pending_cmds: Number of commands already enqueued in the KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ kbdev, \ kcpu_queue, \ @@ -2377,16 +2257,6 @@ struct kbase_tlstream; kcpuq_num_pending_cmds \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ - kbdev, \ - kcpu_queue, \ - kcpu_queue_id, \ - kernel_ctx_id, \ - kcpuq_num_pending_cmds \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE - Delete KCPU Queue @@ -2394,7 +2264,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ kbdev, \ kcpu_queue \ @@ -2407,13 +2276,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL - KCPU Queue enqueues Signal on Fence @@ -2422,7 +2284,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @fence: Fence object handle */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ kbdev, \ kcpu_queue, \ @@ -2437,14 +2298,6 @@ struct kbase_tlstream; fence \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ - kbdev, \ - kcpu_queue, \ - fence \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT - KCPU Queue enqueues Wait on Fence @@ -2453,7 +2306,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @fence: Fence object handle */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ kbdev, \ kcpu_queue, \ @@ -2468,14 +2320,6 @@ struct kbase_tlstream; fence \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ - kbdev, \ - kcpu_queue, \ - fence \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT - KCPU Queue enqueues Wait on Cross Queue Sync Object @@ -2486,7 +2330,6 @@ struct kbase_tlstream; * @compare_value: Semaphore value that should be exceeded for the WAIT to pass * @inherit_error: Flag which indicates if the CQS object error state should be inherited by the queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ kbdev, \ kcpu_queue, \ @@ -2505,16 +2348,6 @@ struct kbase_tlstream; inherit_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ - kbdev, \ - kcpu_queue, \ - cqs_obj_gpu_addr, \ - compare_value, \ - inherit_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET - KCPU Queue enqueues Set on Cross Queue Sync Object @@ -2523,7 +2356,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @cqs_obj_gpu_addr: CQS Object GPU pointer */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( \ kbdev, \ kcpu_queue, \ @@ -2538,14 +2370,6 @@ struct kbase_tlstream; cqs_obj_gpu_addr \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( \ - kbdev, \ - kcpu_queue, \ - cqs_obj_gpu_addr \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT_OPERATION - KCPU Queue enqueues Wait Operation on Cross Queue Sync Object @@ -2558,7 +2382,6 @@ struct kbase_tlstream; * @data_type: Data type of a CQS Object's value * @inherit_error: Flag which indicates if the CQS object error state should be inherited by the queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT_OPERATION( \ kbdev, \ kcpu_queue, \ @@ -2581,18 +2404,6 @@ struct kbase_tlstream; inherit_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT_OPERATION( \ - kbdev, \ - kcpu_queue, \ - cqs_obj_gpu_addr, \ - compare_value, \ - condition, \ - data_type, \ - inherit_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET_OPERATION - KCPU Queue enqueues Set Operation on Cross Queue Sync Object @@ -2604,7 +2415,6 @@ struct kbase_tlstream; * @operation: Operation type performed on semaphore value (SET or ADD) * @data_type: Data type of a CQS Object's value */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET_OPERATION( \ kbdev, \ kcpu_queue, \ @@ -2625,17 +2435,6 @@ struct kbase_tlstream; data_type \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET_OPERATION( \ - kbdev, \ - kcpu_queue, \ - cqs_obj_gpu_addr, \ - value, \ - operation, \ - data_type \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT - KCPU Queue enqueues Map Import @@ -2644,7 +2443,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @map_import_buf_gpu_addr: Map import buffer GPU pointer */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ kbdev, \ kcpu_queue, \ @@ -2659,14 +2457,6 @@ struct kbase_tlstream; map_import_buf_gpu_addr \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ - kbdev, \ - kcpu_queue, \ - map_import_buf_gpu_addr \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT - KCPU Queue enqueues Unmap Import @@ -2675,7 +2465,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @map_import_buf_gpu_addr: Map import buffer GPU pointer */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ kbdev, \ kcpu_queue, \ @@ -2690,14 +2479,6 @@ struct kbase_tlstream; map_import_buf_gpu_addr \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ - kbdev, \ - kcpu_queue, \ - map_import_buf_gpu_addr \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE - KCPU Queue enqueues Unmap Import ignoring reference count @@ -2706,7 +2487,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @map_import_buf_gpu_addr: Map import buffer GPU pointer */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ kbdev, \ kcpu_queue, \ @@ -2721,14 +2501,6 @@ struct kbase_tlstream; map_import_buf_gpu_addr \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ - kbdev, \ - kcpu_queue, \ - map_import_buf_gpu_addr \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC - Begin array of KCPU Queue enqueues JIT Alloc @@ -2736,7 +2508,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ kbdev, \ kcpu_queue \ @@ -2749,13 +2520,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC - Array item of KCPU Queue enqueues JIT Alloc @@ -2772,7 +2536,6 @@ struct kbase_tlstream; * @jit_alloc_flags: Flags specifying the special requirements for the JIT allocation * @jit_alloc_usage_id: A hint about which allocation should be reused. The kernel should attempt to use a previous allocation with the same usage_id */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ kbdev, \ kcpu_queue, \ @@ -2803,22 +2566,6 @@ struct kbase_tlstream; jit_alloc_usage_id \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ - kbdev, \ - kcpu_queue, \ - jit_alloc_gpu_alloc_addr_dest, \ - jit_alloc_va_pages, \ - jit_alloc_commit_pages, \ - jit_alloc_extent, \ - jit_alloc_jit_id, \ - jit_alloc_bin_id, \ - jit_alloc_max_allocations, \ - jit_alloc_flags, \ - jit_alloc_usage_id \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC - End array of KCPU Queue enqueues JIT Alloc @@ -2826,7 +2573,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ kbdev, \ kcpu_queue \ @@ -2839,13 +2585,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE - Begin array of KCPU Queue enqueues JIT Free @@ -2853,7 +2592,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ kbdev, \ kcpu_queue \ @@ -2866,13 +2604,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE - Array item of KCPU Queue enqueues JIT Free @@ -2881,7 +2612,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @jit_alloc_jit_id: Unique ID provided by the caller, this is used to pair allocation and free requests. Zero is not a valid value */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ kbdev, \ kcpu_queue, \ @@ -2896,14 +2626,6 @@ struct kbase_tlstream; jit_alloc_jit_id \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ - kbdev, \ - kcpu_queue, \ - jit_alloc_jit_id \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE - End array of KCPU Queue enqueues JIT Free @@ -2911,7 +2633,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ kbdev, \ kcpu_queue \ @@ -2924,13 +2645,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER - KCPU Queue enqueues Error Barrier @@ -2938,7 +2652,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER( \ kbdev, \ kcpu_queue \ @@ -2951,13 +2664,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND - KCPU Queue enqueues Group Suspend @@ -2967,7 +2673,6 @@ struct kbase_tlstream; * @group_suspend_buf: Pointer to the suspend buffer structure * @gpu_cmdq_grp_handle: GPU Command Queue Group handle which will match userspace */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND( \ kbdev, \ kcpu_queue, \ @@ -2984,15 +2689,6 @@ struct kbase_tlstream; gpu_cmdq_grp_handle \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND( \ - kbdev, \ - kcpu_queue, \ - group_suspend_buf, \ - gpu_cmdq_grp_handle \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START - KCPU Queue starts a Signal on Fence @@ -3000,7 +2696,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ kbdev, \ kcpu_queue \ @@ -3013,13 +2708,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END - KCPU Queue ends a Signal on Fence @@ -3028,7 +2716,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ kbdev, \ kcpu_queue, \ @@ -3043,14 +2730,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START - KCPU Queue starts a Wait on Fence @@ -3058,7 +2737,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ kbdev, \ kcpu_queue \ @@ -3071,13 +2749,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END - KCPU Queue ends a Wait on Fence @@ -3086,7 +2757,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ kbdev, \ kcpu_queue, \ @@ -3101,14 +2771,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START - KCPU Queue starts a Wait on Cross Queue Sync Object @@ -3116,7 +2778,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ kbdev, \ kcpu_queue \ @@ -3129,13 +2790,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END - KCPU Queue ends a Wait on Cross Queue Sync Object @@ -3144,7 +2798,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ kbdev, \ kcpu_queue, \ @@ -3159,14 +2812,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET - KCPU Queue executes a Set on Cross Queue Sync Object @@ -3175,7 +2820,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ kbdev, \ kcpu_queue, \ @@ -3190,14 +2834,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_START - KCPU Queue starts a Wait Operation on Cross Queue Sync Object @@ -3205,7 +2841,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_START( \ kbdev, \ kcpu_queue \ @@ -3218,13 +2853,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_END - KCPU Queue ends a Wait Operation on Cross Queue Sync Object @@ -3233,7 +2861,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_END( \ kbdev, \ kcpu_queue, \ @@ -3248,14 +2875,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_OPERATION_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET_OPERATION - KCPU Queue executes a Set Operation on Cross Queue Sync Object @@ -3264,7 +2883,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET_OPERATION( \ kbdev, \ kcpu_queue, \ @@ -3279,14 +2897,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET_OPERATION( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START - KCPU Queue starts a Map Import @@ -3294,7 +2904,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ kbdev, \ kcpu_queue \ @@ -3307,13 +2916,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END - KCPU Queue ends a Map Import @@ -3322,7 +2924,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ kbdev, \ kcpu_queue, \ @@ -3337,14 +2938,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START - KCPU Queue starts an Unmap Import @@ -3352,7 +2945,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ kbdev, \ kcpu_queue \ @@ -3365,13 +2957,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END - KCPU Queue ends an Unmap Import @@ -3380,7 +2965,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ kbdev, \ kcpu_queue, \ @@ -3395,14 +2979,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START - KCPU Queue starts an Unmap Import ignoring reference count @@ -3410,7 +2986,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ kbdev, \ kcpu_queue \ @@ -3423,13 +2998,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END - KCPU Queue ends an Unmap Import ignoring reference count @@ -3438,7 +3006,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ kbdev, \ kcpu_queue, \ @@ -3453,14 +3020,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START - KCPU Queue starts an array of JIT Allocs @@ -3468,7 +3027,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ kbdev, \ kcpu_queue \ @@ -3481,13 +3039,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - Begin array of KCPU Queue ends an array of JIT Allocs @@ -3495,7 +3046,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ kbdev, \ kcpu_queue \ @@ -3508,13 +3058,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - Array item of KCPU Queue ends an array of JIT Allocs @@ -3525,7 +3068,6 @@ struct kbase_tlstream; * @jit_alloc_gpu_alloc_addr: The JIT allocated GPU virtual address * @jit_alloc_mmu_flags: The MMU flags for the JIT allocation */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ kbdev, \ kcpu_queue, \ @@ -3544,16 +3086,6 @@ struct kbase_tlstream; jit_alloc_mmu_flags \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ - kbdev, \ - kcpu_queue, \ - execute_error, \ - jit_alloc_gpu_alloc_addr, \ - jit_alloc_mmu_flags \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - End array of KCPU Queue ends an array of JIT Allocs @@ -3561,7 +3093,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ kbdev, \ kcpu_queue \ @@ -3574,13 +3105,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START - KCPU Queue starts an array of JIT Frees @@ -3588,7 +3112,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ kbdev, \ kcpu_queue \ @@ -3601,13 +3124,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END - Begin array of KCPU Queue ends an array of JIT Frees @@ -3615,7 +3131,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ kbdev, \ kcpu_queue \ @@ -3628,13 +3143,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END - Array item of KCPU Queue ends an array of JIT Frees @@ -3644,7 +3152,6 @@ struct kbase_tlstream; * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero * @jit_free_pages_used: The actual number of pages used by the JIT allocation */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ kbdev, \ kcpu_queue, \ @@ -3661,15 +3168,6 @@ struct kbase_tlstream; jit_free_pages_used \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ - kbdev, \ - kcpu_queue, \ - execute_error, \ - jit_free_pages_used \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END - End array of KCPU Queue ends an array of JIT Frees @@ -3677,7 +3175,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ kbdev, \ kcpu_queue \ @@ -3690,13 +3187,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER - KCPU Queue executes an Error Barrier @@ -3704,7 +3194,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER( \ kbdev, \ kcpu_queue \ @@ -3717,13 +3206,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START - KCPU Queue starts a group suspend @@ -3731,7 +3213,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @kcpu_queue: KCPU queue */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START( \ kbdev, \ kcpu_queue \ @@ -3744,13 +3225,6 @@ struct kbase_tlstream; kcpu_queue \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END - KCPU Queue ends a group suspend @@ -3759,7 +3233,6 @@ struct kbase_tlstream; * @kcpu_queue: KCPU queue * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END( \ kbdev, \ kcpu_queue, \ @@ -3774,14 +3247,6 @@ struct kbase_tlstream; execute_error \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END( \ - kbdev, \ - kcpu_queue, \ - execute_error \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_L2_CORE_STATE - KBase device updates L2 Core state @@ -3790,7 +3255,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @new_state: New state */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_L2_CORE_STATE( \ kbdev, \ kbase_device_id, \ @@ -3805,14 +3269,6 @@ struct kbase_tlstream; new_state \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_L2_CORE_STATE( \ - kbdev, \ - kbase_device_id, \ - new_state \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_MCU_STATE - KBase device updates MCU state @@ -3821,7 +3277,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @new_state: New state */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_MCU_STATE( \ kbdev, \ kbase_device_id, \ @@ -3836,14 +3291,6 @@ struct kbase_tlstream; new_state \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_MCU_STATE( \ - kbdev, \ - kbase_device_id, \ - new_state \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_DEVICE_SHADER_CORE_STATE - KBase device updates Shader Core state @@ -3852,7 +3299,6 @@ struct kbase_tlstream; * @kbase_device_id: The ID of the physical hardware * @new_state: New state */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_DEVICE_SHADER_CORE_STATE( \ kbdev, \ kbase_device_id, \ @@ -3867,14 +3313,6 @@ struct kbase_tlstream; new_state \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_DEVICE_SHADER_CORE_STATE( \ - kbdev, \ - kbase_device_id, \ - new_state \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_DISABLING - CSF FW is being disabled @@ -3882,7 +3320,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @csffw_cycle: Cycle number of a CSFFW event */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_DISABLING( \ kbdev, \ csffw_cycle \ @@ -3895,13 +3332,6 @@ struct kbase_tlstream; csffw_cycle \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_DISABLING( \ - kbdev, \ - csffw_cycle \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF - CSF FW is off @@ -3909,7 +3339,6 @@ struct kbase_tlstream; * @kbdev: Kbase device * @csffw_cycle: Cycle number of a CSFFW event */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF( \ kbdev, \ csffw_cycle \ @@ -3922,13 +3351,6 @@ struct kbase_tlstream; csffw_cycle \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF( \ - kbdev, \ - csffw_cycle \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW - An overflow has happened with the CSFFW Timeline stream @@ -3937,7 +3359,6 @@ struct kbase_tlstream; * @csffw_timestamp: Timestamp of a CSFFW event * @csffw_cycle: Cycle number of a CSFFW event */ -#if MALI_USE_CSF #define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ kbdev, \ csffw_timestamp, \ @@ -3952,14 +3373,6 @@ struct kbase_tlstream; csffw_cycle \ ); \ } while (0) -#else -#define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ - kbdev, \ - csffw_timestamp, \ - csffw_cycle \ - ) \ - do { } while (0) -#endif /* MALI_USE_CSF */ /** * KBASE_TLSTREAM_AUX_PM_STATE - PM state diff --git a/include/linux/memory_group_manager.h b/include/linux/memory_group_manager.h index ec55d74f56ad..8729e2dfb9e5 100644 --- a/include/linux/memory_group_manager.h +++ b/include/linux/memory_group_manager.h @@ -41,6 +41,11 @@ typedef int vm_fault_t; struct memory_group_manager_device; struct memory_group_manager_import_data; +/** + * enum mgm_pte_flags - Memory Group Manager PTE Flags + * @MMA_VIOLATION: Bit-number of flag used to indicate that Mismatched Memory Attributes (MMA) + * exist for a page. Specifically, this means CPU-uncached, GPU-cached. + */ enum mgm_pte_flags { MMA_VIOLATION = 0, }; @@ -138,11 +143,17 @@ struct memory_group_manager_ops { * decoded to determine the physical address and any other * properties of the mapping the manager requires. * - * This function allows the memory group manager to modify a GPU page - * table entry before it is stored by the kbase module (controller - * driver). It may set certain bits in the page table entry attributes - * or modify the physical address, based on the physical memory group ID, - * PBHA ID, PTE flags and/or additional data in struct memory_group_manager_device. + * This function allows the memory group manager to modify a GPU page table entry before it + * is stored by the kbase module (controller driver). It may set certain bits in the page + * table entry attributes or modify the physical address, based on the physical memory + * group ID, PBHA ID, PTE flags and/or additional data in + * struct memory_group_manager_device. + * + * If mgm_pte_flags has the MMA_VIOLATION bit set - ie, 1< #define KBASE_DUMMY_MODEL_COUNTER_HEADER_DWORDS (4) -#if MALI_USE_CSF #define KBASE_DUMMY_MODEL_COUNTER_PER_CORE (65) -#else /* MALI_USE_CSF */ -#define KBASE_DUMMY_MODEL_COUNTER_PER_CORE (60) -#endif /* MALI_USE_CSF */ #define KBASE_DUMMY_MODEL_COUNTERS_PER_BIT (4) #define KBASE_DUMMY_MODEL_COUNTER_ENABLED(enable_mask, ctr_idx) \ (enable_mask & (1U << (ctr_idx / KBASE_DUMMY_MODEL_COUNTERS_PER_BIT))) diff --git a/include/uapi/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_regmap_jm.h b/include/uapi/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_regmap_jm.h deleted file mode 100644 index d24afccf49ee..000000000000 --- a/include/uapi/gpu/arm/valhall/gpu/backend/mali_kbase_gpu_regmap_jm.h +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _UAPI_KBASE_GPU_REGMAP_JM_H_ -#define _UAPI_KBASE_GPU_REGMAP_JM_H_ - -#endif /* _UAPI_KBASE_GPU_REGMAP_JM_H_ */ diff --git a/include/uapi/gpu/arm/valhall/gpu/mali_kbase_gpu_regmap.h b/include/uapi/gpu/arm/valhall/gpu/mali_kbase_gpu_regmap.h index 8256191f331b..84c3e8c877b7 100644 --- a/include/uapi/gpu/arm/valhall/gpu/mali_kbase_gpu_regmap.h +++ b/include/uapi/gpu/arm/valhall/gpu/mali_kbase_gpu_regmap.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,10 +22,6 @@ #ifndef _UAPI_KBASE_GPU_REGMAP_H_ #define _UAPI_KBASE_GPU_REGMAP_H_ -#if MALI_USE_CSF #include "backend/mali_kbase_gpu_regmap_csf.h" -#else -#include "backend/mali_kbase_gpu_regmap_jm.h" -#endif /* !MALI_USE_CSF */ #endif /* _UAPI_KBASE_GPU_REGMAP_H_ */ diff --git a/include/uapi/gpu/arm/valhall/jm/mali_base_jm_kernel.h b/include/uapi/gpu/arm/valhall/jm/mali_base_jm_kernel.h deleted file mode 100644 index 48a43210d03f..000000000000 --- a/include/uapi/gpu/arm/valhall/jm/mali_base_jm_kernel.h +++ /dev/null @@ -1,892 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _UAPI_BASE_JM_KERNEL_H_ -#define _UAPI_BASE_JM_KERNEL_H_ - -#include -#include "../mali_base_common_kernel.h" - -/* Similar to BASE_MEM_TILER_ALIGN_TOP, memory starting from the end of the - * initial commit is aligned to 'extension' pages, where 'extension' must be a power - * of 2 and no more than BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES - */ -#define BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP (1 << 0) - -/** - * BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE - If set, the heap info address points - * to a __u32 holding the used size in bytes; - * otherwise it points to a __u64 holding the lowest address of unused memory. - */ -#define BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE (1 << 1) - -/** - * BASE_JIT_ALLOC_VALID_FLAGS - Valid set of just-in-time memory allocation flags - * - * Note: BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE cannot be set if heap_info_gpu_addr - * in %base_jit_alloc_info is 0 (atom with BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE set - * and heap_info_gpu_addr being 0 will be rejected). - */ -#define BASE_JIT_ALLOC_VALID_FLAGS \ - (BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP | BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) - -/* Bitpattern describing the ::base_context_create_flags that can be - * passed to base_context_init() - */ -#define BASEP_CONTEXT_CREATE_ALLOWED_FLAGS \ - (BASE_CONTEXT_CCTX_EMBEDDED | BASEP_CONTEXT_CREATE_KERNEL_FLAGS) - -/* - * Private flags used on the base context - * - * These start at bit 31, and run down to zero. - * - * They share the same space as base_context_create_flags, and so must - * not collide with them. - */ - -/* Private flag tracking whether job descriptor dumping is disabled */ -#define BASEP_CONTEXT_FLAG_JOB_DUMP_DISABLED ((base_context_create_flags)(1 << 31)) - -/* Flags for base tracepoint specific to JM */ -#define BASE_TLSTREAM_FLAGS_MASK \ - (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | BASE_TLSTREAM_JOB_DUMPING_ENABLED) -/* - * Dependency stuff, keep it private for now. May want to expose it if - * we decide to make the number of semaphores a configurable - * option. - */ -#define BASE_JD_ATOM_COUNT 256 - -/* Set/reset values for a software event */ -#define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) -#define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) - -/** - * struct base_jd_udata - Per-job data - * - * @blob: per-job data array - * - * This structure is used to store per-job data, and is completely unused - * by the Base driver. It can be used to store things such as callback - * function pointer, data to handle job completion. It is guaranteed to be - * untouched by the Base driver. - */ -struct base_jd_udata { - __u64 blob[2]; -}; - -/** - * typedef base_jd_dep_type - Job dependency type. - * - * A flags field will be inserted into the atom structure to specify whether a - * dependency is a data or ordering dependency (by putting it before/after - * 'core_req' in the structure it should be possible to add without changing - * the structure size). - * When the flag is set for a particular dependency to signal that it is an - * ordering only dependency then errors will not be propagated. - */ -typedef __u8 base_jd_dep_type; - -#define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */ -#define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */ -#define BASE_JD_DEP_TYPE_ORDER (1U << 1) /**< Order dependency */ - -/** - * typedef base_jd_core_req - Job chain hardware requirements. - * - * A job chain must specify what GPU features it needs to allow the - * driver to schedule the job correctly. By not specifying the - * correct settings can/will cause an early job termination. Multiple - * values can be ORed together to specify multiple requirements. - * Special case is ::BASE_JD_REQ_DEP, which is used to express complex - * dependencies, and that doesn't execute anything on the hardware. - */ -typedef __u32 base_jd_core_req; - -/* Requirements that come from the HW */ - -/* No requirement, dependency only - */ -#define BASE_JD_REQ_DEP ((base_jd_core_req)0) - -/* Requires fragment shaders - */ -#define BASE_JD_REQ_FS ((base_jd_core_req)1 << 0) - -/* Requires compute shaders - * - * This covers any of the following GPU job types: - * - Vertex Shader Job - * - Geometry Shader Job - * - An actual Compute Shader Job - * - * Compare this with BASE_JD_REQ_ONLY_COMPUTE, which specifies that the - * job is specifically just the "Compute Shader" job type, and not the "Vertex - * Shader" nor the "Geometry Shader" job type. - */ -#define BASE_JD_REQ_CS ((base_jd_core_req)1 << 1) - -/* Requires tiling */ -#define BASE_JD_REQ_T ((base_jd_core_req)1 << 2) - -/* Requires cache flushes */ -#define BASE_JD_REQ_CF ((base_jd_core_req)1 << 3) - -/* Requires value writeback */ -#define BASE_JD_REQ_V ((base_jd_core_req)1 << 4) - -/* SW-only requirements - the HW does not expose these as part of the job slot - * capabilities - */ - -/* Requires fragment job with AFBC encoding */ -#define BASE_JD_REQ_FS_AFBC ((base_jd_core_req)1 << 13) - -/* SW-only requirement: coalesce completion events. - * If this bit is set then completion of this atom will not cause an event to - * be sent to userspace, whether successful or not; completion events will be - * deferred until an atom completes which does not have this bit set. - * - * This bit may not be used in combination with BASE_JD_REQ_EXTERNAL_RESOURCES. - */ -#define BASE_JD_REQ_EVENT_COALESCE ((base_jd_core_req)1 << 5) - -/* SW Only requirement: the job chain requires a coherent core group. We don't - * mind which coherent core group is used. - */ -#define BASE_JD_REQ_COHERENT_GROUP ((base_jd_core_req)1 << 6) - -/* SW Only requirement: The performance counters should be enabled only when - * they are needed, to reduce power consumption. - */ -#define BASE_JD_REQ_PERMON ((base_jd_core_req)1 << 7) - -/* SW Only requirement: External resources are referenced by this atom. - * - * This bit may not be used in combination with BASE_JD_REQ_EVENT_COALESCE and - * BASE_JD_REQ_SOFT_EVENT_WAIT. - */ -#define BASE_JD_REQ_EXTERNAL_RESOURCES ((base_jd_core_req)1 << 8) - -/* SW Only requirement: Software defined job. Jobs with this bit set will not be - * submitted to the hardware but will cause some action to happen within the - * driver - */ -#define BASE_JD_REQ_SOFT_JOB ((base_jd_core_req)1 << 9) - -#define BASE_JD_REQ_SOFT_DUMP_CPU_GPU_TIME (BASE_JD_REQ_SOFT_JOB | 0x1) -#define BASE_JD_REQ_SOFT_FENCE_TRIGGER (BASE_JD_REQ_SOFT_JOB | 0x2) -#define BASE_JD_REQ_SOFT_FENCE_WAIT (BASE_JD_REQ_SOFT_JOB | 0x3) - -/* 0x4 RESERVED for now */ - -/* SW only requirement: event wait/trigger job. - * - * - BASE_JD_REQ_SOFT_EVENT_WAIT: this job will block until the event is set. - * - BASE_JD_REQ_SOFT_EVENT_SET: this job sets the event, thus unblocks the - * other waiting jobs. It completes immediately. - * - BASE_JD_REQ_SOFT_EVENT_RESET: this job resets the event, making it - * possible for other jobs to wait upon. It completes immediately. - */ -#define BASE_JD_REQ_SOFT_EVENT_WAIT (BASE_JD_REQ_SOFT_JOB | 0x5) -#define BASE_JD_REQ_SOFT_EVENT_SET (BASE_JD_REQ_SOFT_JOB | 0x6) -#define BASE_JD_REQ_SOFT_EVENT_RESET (BASE_JD_REQ_SOFT_JOB | 0x7) - -#define BASE_JD_REQ_SOFT_DEBUG_COPY (BASE_JD_REQ_SOFT_JOB | 0x8) - -/* SW only requirement: Just In Time allocation - * - * This job requests a single or multiple just-in-time allocations through a - * list of base_jit_alloc_info structure which is passed via the jc element of - * the atom. The number of base_jit_alloc_info structures present in the - * list is passed via the nr_extres element of the atom - * - * It should be noted that the id entry in base_jit_alloc_info must not - * be reused until it has been released via BASE_JD_REQ_SOFT_JIT_FREE. - * - * Should this soft job fail it is expected that a BASE_JD_REQ_SOFT_JIT_FREE - * soft job to free the JIT allocation is still made. - * - * The job will complete immediately. - */ -#define BASE_JD_REQ_SOFT_JIT_ALLOC (BASE_JD_REQ_SOFT_JOB | 0x9) - -/* SW only requirement: Just In Time free - * - * This job requests a single or multiple just-in-time allocations created by - * BASE_JD_REQ_SOFT_JIT_ALLOC to be freed. The ID list of the just-in-time - * allocations is passed via the jc element of the atom. - * - * The job will complete immediately. - */ -#define BASE_JD_REQ_SOFT_JIT_FREE (BASE_JD_REQ_SOFT_JOB | 0xa) - -/* SW only requirement: Map external resource - * - * This job requests external resource(s) are mapped once the dependencies - * of the job have been satisfied. The list of external resources are - * passed via the jc element of the atom which is a pointer to a - * base_external_resource_list. - */ -#define BASE_JD_REQ_SOFT_EXT_RES_MAP (BASE_JD_REQ_SOFT_JOB | 0xb) - -/* SW only requirement: Unmap external resource - * - * This job requests external resource(s) are unmapped once the dependencies - * of the job has been satisfied. The list of external resources are - * passed via the jc element of the atom which is a pointer to a - * base_external_resource_list. - */ -#define BASE_JD_REQ_SOFT_EXT_RES_UNMAP (BASE_JD_REQ_SOFT_JOB | 0xc) - -/* HW Requirement: Requires Compute shaders (but not Vertex or Geometry Shaders) - * - * This indicates that the Job Chain contains GPU jobs of the 'Compute - * Shaders' type. - * - * In contrast to BASE_JD_REQ_CS, this does not indicate that the Job - * Chain contains 'Geometry Shader' or 'Vertex Shader' jobs. - */ -#define BASE_JD_REQ_ONLY_COMPUTE ((base_jd_core_req)1 << 10) - -/* HW Requirement: Use the base_jd_atom::device_nr field to specify a - * particular core group - * - * If both BASE_JD_REQ_COHERENT_GROUP and this flag are set, this flag - * takes priority - * - * This is only guaranteed to work for BASE_JD_REQ_ONLY_COMPUTE atoms. - */ -#define BASE_JD_REQ_SPECIFIC_COHERENT_GROUP ((base_jd_core_req)1 << 11) - -/* SW Flag: If this bit is set then the successful completion of this atom - * will not cause an event to be sent to userspace - */ -#define BASE_JD_REQ_EVENT_ONLY_ON_FAILURE ((base_jd_core_req)1 << 12) - -/* SW Flag: If this bit is set then completion of this atom will not cause an - * event to be sent to userspace, whether successful or not. - */ -#define BASEP_JD_REQ_EVENT_NEVER ((base_jd_core_req)1 << 14) - -/* SW Flag: Skip GPU cache clean and invalidation before starting a GPU job. - * - * If this bit is set then the GPU's cache will not be cleaned and invalidated - * until a GPU job starts which does not have this bit set or a job completes - * which does not have the BASE_JD_REQ_SKIP_CACHE_END bit set. Do not use - * if the CPU may have written to memory addressed by the job since the last job - * without this bit set was submitted. - */ -#define BASE_JD_REQ_SKIP_CACHE_START ((base_jd_core_req)1 << 15) - -/* SW Flag: Skip GPU cache clean and invalidation after a GPU job completes. - * - * If this bit is set then the GPU's cache will not be cleaned and invalidated - * until a GPU job completes which does not have this bit set or a job starts - * which does not have the BASE_JD_REQ_SKIP_CACHE_START bit set. Do not use - * if the CPU may read from or partially overwrite memory addressed by the job - * before the next job without this bit set completes. - */ -#define BASE_JD_REQ_SKIP_CACHE_END ((base_jd_core_req)1 << 16) - -/* Request the atom be executed on a specific job slot. - * - * When this flag is specified, it takes precedence over any existing job slot - * selection logic. - */ -#define BASE_JD_REQ_JOB_SLOT ((base_jd_core_req)1 << 17) - -/* SW-only requirement: The atom needs to run on a limited core mask affinity. - * - * If this bit is set then the kbase_context.limited_core_mask will be applied - * to the affinity. - */ -#define BASE_JD_REQ_LIMITED_CORE_MASK ((base_jd_core_req)1 << 20) - -/* These requirement bits are currently unused in base_jd_core_req - */ -#define BASEP_JD_REQ_RESERVED \ - (~(BASE_JD_REQ_ATOM_TYPE | BASE_JD_REQ_EXTERNAL_RESOURCES | \ - BASE_JD_REQ_EVENT_ONLY_ON_FAILURE | BASEP_JD_REQ_EVENT_NEVER | \ - BASE_JD_REQ_EVENT_COALESCE | BASE_JD_REQ_COHERENT_GROUP | \ - BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ - BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END | BASE_JD_REQ_JOB_SLOT | \ - BASE_JD_REQ_LIMITED_CORE_MASK)) - -/* Mask of all bits in base_jd_core_req that control the type of the atom. - * - * This allows dependency only atoms to have flags set - */ -#define BASE_JD_REQ_ATOM_TYPE \ - (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T | BASE_JD_REQ_CF | BASE_JD_REQ_V | \ - BASE_JD_REQ_SOFT_JOB | BASE_JD_REQ_ONLY_COMPUTE) - -/** - * BASE_JD_REQ_SOFT_JOB_TYPE - Mask of all bits in base_jd_core_req that - * controls the type of a soft job. - */ -#define BASE_JD_REQ_SOFT_JOB_TYPE (BASE_JD_REQ_SOFT_JOB | 0x1f) - -/* Returns non-zero value if core requirements passed define a soft job or - * a dependency only job. - */ -#define BASE_JD_REQ_SOFT_JOB_OR_DEP(core_req) \ - (((core_req)&BASE_JD_REQ_SOFT_JOB) || ((core_req)&BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) - -/** - * enum kbase_jd_atom_state - Atom states - * - * @KBASE_JD_ATOM_STATE_UNUSED: Atom is not used. - * @KBASE_JD_ATOM_STATE_QUEUED: Atom is queued in JD. - * @KBASE_JD_ATOM_STATE_IN_JS: Atom has been given to JS (is runnable/running). - * @KBASE_JD_ATOM_STATE_HW_COMPLETED: Atom has been completed, but not yet - * handed back to job dispatcher for - * dependency resolution. - * @KBASE_JD_ATOM_STATE_COMPLETED: Atom has been completed, but not yet handed - * back to userspace. - */ -enum kbase_jd_atom_state { - KBASE_JD_ATOM_STATE_UNUSED, - KBASE_JD_ATOM_STATE_QUEUED, - KBASE_JD_ATOM_STATE_IN_JS, - KBASE_JD_ATOM_STATE_HW_COMPLETED, - KBASE_JD_ATOM_STATE_COMPLETED -}; - -/** - * typedef base_atom_id - Type big enough to store an atom number in. - */ -typedef __u8 base_atom_id; - -/** - * struct base_dependency - base dependency - * - * @atom_id: An atom number - * @dependency_type: Dependency type - */ -struct base_dependency { - base_atom_id atom_id; - base_jd_dep_type dependency_type; -}; - -/** - * typedef base_jd_prio - Base Atom priority. - * - * Only certain priority levels are actually implemented, as specified by the - * BASE_JD_PRIO_<...> definitions below. It is undefined to use a priority - * level that is not one of those defined below. - * - * Priority levels only affect scheduling after the atoms have had dependencies - * resolved. For example, a low priority atom that has had its dependencies - * resolved might run before a higher priority atom that has not had its - * dependencies resolved. - * - * In general, fragment atoms do not affect non-fragment atoms with - * lower priorities, and vice versa. One exception is that there is only one - * priority value for each context. So a high-priority (e.g.) fragment atom - * could increase its context priority, causing its non-fragment atoms to also - * be scheduled sooner. - * - * The atoms are scheduled as follows with respect to their priorities: - * * Let atoms 'X' and 'Y' be for the same job slot who have dependencies - * resolved, and atom 'X' has a higher priority than atom 'Y' - * * If atom 'Y' is currently running on the HW, then it is interrupted to - * allow atom 'X' to run soon after - * * If instead neither atom 'Y' nor atom 'X' are running, then when choosing - * the next atom to run, atom 'X' will always be chosen instead of atom 'Y' - * * Any two atoms that have the same priority could run in any order with - * respect to each other. That is, there is no ordering constraint between - * atoms of the same priority. - * - * The sysfs file 'js_ctx_scheduling_mode' is used to control how atoms are - * scheduled between contexts. The default value, 0, will cause higher-priority - * atoms to be scheduled first, regardless of their context. The value 1 will - * use a round-robin algorithm when deciding which context's atoms to schedule - * next, so higher-priority atoms can only preempt lower priority atoms within - * the same context. See KBASE_JS_SYSTEM_PRIORITY_MODE and - * KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE for more details. - */ -typedef __u8 base_jd_prio; - -/* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */ -#define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0) -/* High atom priority. This is a priority higher than BASE_JD_PRIO_MEDIUM and - * BASE_JD_PRIO_LOW - */ -#define BASE_JD_PRIO_HIGH ((base_jd_prio)1) -/* Low atom priority. */ -#define BASE_JD_PRIO_LOW ((base_jd_prio)2) -/* Real-Time atom priority. This is a priority higher than BASE_JD_PRIO_HIGH, - * BASE_JD_PRIO_MEDIUM, and BASE_JD_PRIO_LOW - */ -#define BASE_JD_PRIO_REALTIME ((base_jd_prio)3) - -/* Invalid atom priority (max uint8_t value) */ -#define BASE_JD_PRIO_INVALID ((base_jd_prio)255) - -/* Count of the number of priority levels. This itself is not a valid - * base_jd_prio setting - */ -#define BASE_JD_NR_PRIO_LEVELS 4 - -/** - * struct base_jd_atom_v2 - Node of a dependency graph used to submit a - * GPU job chain or soft-job to the kernel driver. - * - * @jc: GPU address of a job chain. - * @udata: User data. - * @extres_list: List of external resources. - * @nr_extres: Number of external resources or JIT allocations. - * @jit_id: Zero-terminated array of IDs of just-in-time memory - * allocations written to by the atom. When the atom - * completes, the value stored at the - * &struct_base_jit_alloc_info.heap_info_gpu_addr of - * each allocation is read in order to enforce an - * overall physical memory usage limit. - * @pre_dep: Pre-dependencies. One need to use SETTER function to assign - * this field; this is done in order to reduce possibility of - * improper assignment of a dependency field. - * @atom_number: Unique number to identify the atom. - * @prio: Atom priority. Refer to base_jd_prio for more details. - * @device_nr: Core group when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP - * specified. - * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. - * @core_req: Core requirements. - * @padding: Unused. Must be zero. - * - * This structure has changed since UK 10.2 for which base_jd_core_req was a - * __u16 value. - * - * In UK 10.3 a core_req field of a __u32 type was added to the end of the - * structure, and the place in the structure previously occupied by __u16 - * core_req was kept but renamed to compat_core_req. - * - * From UK 11.20 - compat_core_req is now occupied by __u8 jit_id[2]. - * Compatibility with UK 10.x from UK 11.y is not handled because - * the major version increase prevents this. - * - * For UK 11.20 jit_id[2] must be initialized to zero. - */ -struct base_jd_atom_v2 { - __u64 jc; - struct base_jd_udata udata; - __u64 extres_list; - __u16 nr_extres; - __u8 jit_id[2]; - struct base_dependency pre_dep[2]; - base_atom_id atom_number; - base_jd_prio prio; - __u8 device_nr; - __u8 jobslot; - base_jd_core_req core_req; - __u8 padding[8]; -}; - -/** - * struct base_jd_atom - Same as base_jd_atom_v2, but has an extra seq_nr - * at the beginning. - * - * @seq_nr: Sequence number of logical grouping of atoms. - * @jc: GPU address of a job chain. - * @udata: User data. - * @extres_list: List of external resources. - * @nr_extres: Number of external resources or JIT allocations. - * @jit_id: Zero-terminated array of IDs of just-in-time memory - * allocations written to by the atom. When the atom - * completes, the value stored at the - * &struct_base_jit_alloc_info.heap_info_gpu_addr of - * each allocation is read in order to enforce an - * overall physical memory usage limit. - * @pre_dep: Pre-dependencies. One need to use SETTER function to assign - * this field; this is done in order to reduce possibility of - * improper assignment of a dependency field. - * @atom_number: Unique number to identify the atom. - * @prio: Atom priority. Refer to base_jd_prio for more details. - * @device_nr: Core group when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP - * specified. - * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. - * @core_req: Core requirements. - * @renderpass_id: Renderpass identifier used to associate an atom that has - * BASE_JD_REQ_START_RENDERPASS set in its core requirements - * with an atom that has BASE_JD_REQ_END_RENDERPASS set. - * @padding: Unused. Must be zero. - */ -typedef struct base_jd_atom { - __u64 seq_nr; - __u64 jc; - struct base_jd_udata udata; - __u64 extres_list; - __u16 nr_extres; - __u8 jit_id[2]; - struct base_dependency pre_dep[2]; - base_atom_id atom_number; - base_jd_prio prio; - __u8 device_nr; - __u8 jobslot; - base_jd_core_req core_req; - __u8 renderpass_id; - __u8 padding[7]; -} base_jd_atom; - -/* Job chain event code bits - * Defines the bits used to create ::base_jd_event_code - */ -enum { - BASE_JD_SW_EVENT_KERNEL = (1u << 15), /* Kernel side event */ - BASE_JD_SW_EVENT = (1u << 14), /* SW defined event */ - /* Event indicates success (SW events only) */ - BASE_JD_SW_EVENT_SUCCESS = (1u << 13), - BASE_JD_SW_EVENT_JOB = (0u << 11), /* Job related event */ - BASE_JD_SW_EVENT_BAG = (1u << 11), /* Bag related event */ - BASE_JD_SW_EVENT_INFO = (2u << 11), /* Misc/info event */ - BASE_JD_SW_EVENT_RESERVED = (3u << 11), /* Reserved event type */ - /* Mask to extract the type from an event code */ - BASE_JD_SW_EVENT_TYPE_MASK = (3u << 11) -}; - -/** - * enum base_jd_event_code - Job chain event codes - * - * @BASE_JD_EVENT_RANGE_HW_NONFAULT_START: Start of hardware non-fault status - * codes. - * Obscurely, BASE_JD_EVENT_TERMINATED - * indicates a real fault, because the - * job was hard-stopped. - * @BASE_JD_EVENT_NOT_STARTED: Can't be seen by userspace, treated as - * 'previous job done'. - * @BASE_JD_EVENT_STOPPED: Can't be seen by userspace, becomes - * TERMINATED, DONE or JOB_CANCELLED. - * @BASE_JD_EVENT_TERMINATED: This is actually a fault status code - the job - * was hard stopped. - * @BASE_JD_EVENT_ACTIVE: Can't be seen by userspace, jobs only returned on - * complete/fail/cancel. - * @BASE_JD_EVENT_RANGE_HW_NONFAULT_END: End of hardware non-fault status codes. - * Obscurely, BASE_JD_EVENT_TERMINATED - * indicates a real fault, - * because the job was hard-stopped. - * @BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START: Start of hardware fault and - * software error status codes. - * @BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END: End of hardware fault and - * software error status codes. - * @BASE_JD_EVENT_RANGE_SW_SUCCESS_START: Start of software success status - * codes. - * @BASE_JD_EVENT_RANGE_SW_SUCCESS_END: End of software success status codes. - * @BASE_JD_EVENT_RANGE_KERNEL_ONLY_START: Start of kernel-only status codes. - * Such codes are never returned to - * user-space. - * @BASE_JD_EVENT_RANGE_KERNEL_ONLY_END: End of kernel-only status codes. - * @BASE_JD_EVENT_DONE: atom has completed successfull - * @BASE_JD_EVENT_JOB_CONFIG_FAULT: Atom dependencies configuration error which - * shall result in a failed atom - * @BASE_JD_EVENT_JOB_POWER_FAULT: The job could not be executed because the - * part of the memory system required to access - * job descriptors was not powered on - * @BASE_JD_EVENT_JOB_READ_FAULT: Reading a job descriptor into the Job - * manager failed - * @BASE_JD_EVENT_JOB_WRITE_FAULT: Writing a job descriptor from the Job - * manager failed - * @BASE_JD_EVENT_JOB_AFFINITY_FAULT: The job could not be executed because the - * specified affinity mask does not intersect - * any available cores - * @BASE_JD_EVENT_JOB_BUS_FAULT: A bus access failed while executing a job - * @BASE_JD_EVENT_INSTR_INVALID_PC: A shader instruction with an illegal program - * counter was executed. - * @BASE_JD_EVENT_INSTR_INVALID_ENC: A shader instruction with an illegal - * encoding was executed. - * @BASE_JD_EVENT_INSTR_TYPE_MISMATCH: A shader instruction was executed where - * the instruction encoding did not match the - * instruction type encoded in the program - * counter. - * @BASE_JD_EVENT_INSTR_OPERAND_FAULT: A shader instruction was executed that - * contained invalid combinations of operands. - * @BASE_JD_EVENT_INSTR_TLS_FAULT: A shader instruction was executed that tried - * to access the thread local storage section - * of another thread. - * @BASE_JD_EVENT_INSTR_ALIGN_FAULT: A shader instruction was executed that - * tried to do an unsupported unaligned memory - * access. - * @BASE_JD_EVENT_INSTR_BARRIER_FAULT: A shader instruction was executed that - * failed to complete an instruction barrier. - * @BASE_JD_EVENT_DATA_INVALID_FAULT: Any data structure read as part of the job - * contains invalid combinations of data. - * @BASE_JD_EVENT_TILE_RANGE_FAULT: Tile or fragment shading was asked to - * process a tile that is entirely outside the - * bounding box of the frame. - * @BASE_JD_EVENT_STATE_FAULT: Matches ADDR_RANGE_FAULT. A virtual address - * has been found that exceeds the virtual - * address range. - * @BASE_JD_EVENT_OUT_OF_MEMORY: The tiler ran out of memory when executing a job. - * @BASE_JD_EVENT_UNKNOWN: If multiple jobs in a job chain fail, only - * the first one the reports an error will set - * and return full error information. - * Subsequent failing jobs will not update the - * error status registers, and may write an - * error status of UNKNOWN. - * @BASE_JD_EVENT_DELAYED_BUS_FAULT: The GPU received a bus fault for access to - * physical memory where the original virtual - * address is no longer available. - * @BASE_JD_EVENT_SHAREABILITY_FAULT: Matches GPU_SHAREABILITY_FAULT. A cache - * has detected that the same line has been - * accessed as both shareable and non-shareable - * memory from inside the GPU. - * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1: A memory access hit an invalid table - * entry at level 1 of the translation table. - * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2: A memory access hit an invalid table - * entry at level 2 of the translation table. - * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3: A memory access hit an invalid table - * entry at level 3 of the translation table. - * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4: A memory access hit an invalid table - * entry at level 4 of the translation table. - * @BASE_JD_EVENT_PERMISSION_FAULT: A memory access could not be allowed due to - * the permission flags set in translation - * table - * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1: A bus fault occurred while reading - * level 0 of the translation tables. - * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2: A bus fault occurred while reading - * level 1 of the translation tables. - * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3: A bus fault occurred while reading - * level 2 of the translation tables. - * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4: A bus fault occurred while reading - * level 3 of the translation tables. - * @BASE_JD_EVENT_ACCESS_FLAG: Matches ACCESS_FLAG_0. A memory access hit a - * translation table entry with the ACCESS_FLAG - * bit set to zero in level 0 of the - * page table, and the DISABLE_AF_FAULT flag - * was not set. - * @BASE_JD_EVENT_MEM_GROWTH_FAILED: raised for JIT_ALLOC atoms that failed to - * grow memory on demand - * @BASE_JD_EVENT_JOB_CANCELLED: raised when this atom was hard-stopped or its - * dependencies failed - * @BASE_JD_EVENT_JOB_INVALID: raised for many reasons, including invalid data - * in the atom which overlaps with - * BASE_JD_EVENT_JOB_CONFIG_FAULT, or if the - * platform doesn't support the feature specified in - * the atom. - * @BASE_JD_EVENT_DRV_TERMINATED: this is a special event generated to indicate - * to userspace that the KBase context has been - * destroyed and Base should stop listening for - * further events - * @BASE_JD_EVENT_REMOVED_FROM_NEXT: raised when an atom that was configured in - * the GPU has to be retried (but it has not - * started) due to e.g., GPU reset - * HW and low-level SW events are represented by event codes. - * The status of jobs which succeeded are also represented by - * an event code (see @BASE_JD_EVENT_DONE). - * Events are usually reported as part of a &struct base_jd_event. - * - * The event codes are encoded in the following way: - * * 10:0 - subtype - * * 12:11 - type - * * 13 - SW success (only valid if the SW bit is set) - * * 14 - SW event (HW event if not set) - * * 15 - Kernel event (should never be seen in userspace) - * - * Events are split up into ranges as follows: - * * BASE_JD_EVENT_RANGE__START - * * BASE_JD_EVENT_RANGE__END - * - * code is in 's range when: - * BASE_JD_EVENT_RANGE__START <= code < - * BASE_JD_EVENT_RANGE__END - * - * Ranges can be asserted for adjacency by testing that the END of the previous - * is equal to the START of the next. This is useful for optimizing some tests - * for range. - * - * A limitation is that the last member of this enum must explicitly be handled - * (with an assert-unreachable statement) in switch statements that use - * variables of this type. Otherwise, the compiler warns that we have not - * handled that enum value. - */ -enum base_jd_event_code { - /* HW defined exceptions */ - BASE_JD_EVENT_RANGE_HW_NONFAULT_START = 0, - - /* non-fatal exceptions */ - BASE_JD_EVENT_NOT_STARTED = 0x00, - BASE_JD_EVENT_DONE = 0x01, - BASE_JD_EVENT_STOPPED = 0x03, - BASE_JD_EVENT_TERMINATED = 0x04, - BASE_JD_EVENT_ACTIVE = 0x08, - - BASE_JD_EVENT_RANGE_HW_NONFAULT_END = 0x40, - BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_START = 0x40, - - /* job exceptions */ - BASE_JD_EVENT_JOB_CONFIG_FAULT = 0x40, - BASE_JD_EVENT_JOB_POWER_FAULT = 0x41, - BASE_JD_EVENT_JOB_READ_FAULT = 0x42, - BASE_JD_EVENT_JOB_WRITE_FAULT = 0x43, - BASE_JD_EVENT_JOB_AFFINITY_FAULT = 0x44, - BASE_JD_EVENT_JOB_BUS_FAULT = 0x48, - BASE_JD_EVENT_INSTR_INVALID_PC = 0x50, - BASE_JD_EVENT_INSTR_INVALID_ENC = 0x51, - BASE_JD_EVENT_INSTR_TYPE_MISMATCH = 0x52, - BASE_JD_EVENT_INSTR_OPERAND_FAULT = 0x53, - BASE_JD_EVENT_INSTR_TLS_FAULT = 0x54, - BASE_JD_EVENT_INSTR_BARRIER_FAULT = 0x55, - BASE_JD_EVENT_INSTR_ALIGN_FAULT = 0x56, - BASE_JD_EVENT_DATA_INVALID_FAULT = 0x58, - BASE_JD_EVENT_TILE_RANGE_FAULT = 0x59, - BASE_JD_EVENT_STATE_FAULT = 0x5A, - BASE_JD_EVENT_OUT_OF_MEMORY = 0x60, - BASE_JD_EVENT_UNKNOWN = 0x7F, - - /* GPU exceptions */ - BASE_JD_EVENT_DELAYED_BUS_FAULT = 0x80, - BASE_JD_EVENT_SHAREABILITY_FAULT = 0x88, - - /* MMU exceptions */ - BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1 = 0xC1, - BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2 = 0xC2, - BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3 = 0xC3, - BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4 = 0xC4, - BASE_JD_EVENT_PERMISSION_FAULT = 0xC8, - BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1 = 0xD1, - BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2 = 0xD2, - BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3 = 0xD3, - BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4 = 0xD4, - BASE_JD_EVENT_ACCESS_FLAG = 0xD8, - - /* SW defined exceptions */ - BASE_JD_EVENT_MEM_GROWTH_FAILED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x000, - BASE_JD_EVENT_JOB_CANCELLED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x002, - BASE_JD_EVENT_JOB_INVALID = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_JOB | 0x003, - - BASE_JD_EVENT_RANGE_HW_FAULT_OR_SW_ERROR_END = BASE_JD_SW_EVENT | - BASE_JD_SW_EVENT_RESERVED | 0x3FF, - - BASE_JD_EVENT_RANGE_SW_SUCCESS_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | 0x000, - - BASE_JD_EVENT_DRV_TERMINATED = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | - BASE_JD_SW_EVENT_INFO | 0x000, - - BASE_JD_EVENT_RANGE_SW_SUCCESS_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_SUCCESS | - BASE_JD_SW_EVENT_RESERVED | 0x3FF, - - BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | 0x000, - BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | - BASE_JD_SW_EVENT_JOB | 0x000, - - BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | - BASE_JD_SW_EVENT_RESERVED | 0x3FF -}; - -/** - * struct base_jd_event_v2 - Event reporting structure - * - * @event_code: event code of type @ref base_jd_event_code. - * @atom_number: the atom number that has completed. - * @padding: padding. - * @udata: user data. - * - * This structure is used by the kernel driver to report information - * about GPU events. They can either be HW-specific events or low-level - * SW events, such as job-chain completion. - * - * The event code contains an event type field which can be extracted - * by ANDing with BASE_JD_SW_EVENT_TYPE_MASK. - */ -struct base_jd_event_v2 { - __u32 event_code; - base_atom_id atom_number; - __u8 padding[3]; - struct base_jd_udata udata; -}; - -/** - * struct base_dump_cpu_gpu_counters - Structure for - * BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS - * jobs. - * @system_time: gpu timestamp - * @cycle_counter: gpu cycle count - * @sec: cpu time(sec) - * @usec: cpu time(usec) - * @padding: padding - * - * This structure is stored into the memory pointed to by the @jc field - * of &struct base_jd_atom. - * - * It must not occupy the same CPU cache line(s) as any neighboring data. - * This is to avoid cases where access to pages containing the structure - * is shared between cached and un-cached memory regions, which would - * cause memory corruption. - */ - -struct base_dump_cpu_gpu_counters { - __u64 system_time; - __u64 cycle_counter; - __u64 sec; - __u32 usec; - __u8 padding[36]; -}; - -/** - * struct mali_base_gpu_core_props - GPU core props info - * - * @product_id: Pro specific value. - * @version_status: Status of the GPU release. No defined values, but starts at - * 0 and increases by one for each release status (alpha, beta, EAC, etc.). - * 4 bit values (0-15). - * @minor_revision: Minor release number of the GPU. "P" part of an "RnPn" - * release number. - * 8 bit values (0-255). - * @major_revision: Major release number of the GPU. "R" part of an "RnPn" - * release number. - * 4 bit values (0-15). - * @padding: padding to align to 8-byte - * @gpu_freq_khz_max: The maximum GPU frequency. Reported to applications by - * clGetDeviceInfo() - * @log2_program_counter_size: Size of the shader program counter, in bits. - * @texture_features: TEXTURE_FEATURES_x registers, as exposed by the GPU. This - * is a bitpattern where a set bit indicates that the format is supported. - * Before using a texture format, it is recommended that the corresponding - * bit be checked. - * @paddings_1: Padding bytes. - * @gpu_available_memory_size: Theoretical maximum memory available to the GPU. - * It is unlikely that a client will be able to allocate all of this memory - * for their own purposes, but this at least provides an upper bound on the - * memory available to the GPU. - * This is required for OpenCL's clGetDeviceInfo() call when - * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The - * client will not be expecting to allocate anywhere near this value. - * @num_exec_engines: The number of execution engines. Only valid for tGOX - * (Bifrost) GPUs, where GPU_HAS_REG_CORE_FEATURES is defined. Otherwise, - * this is always 0. - * @paddings_2: Padding bytes. - */ -struct mali_base_gpu_core_props { - __u32 product_id; - __u16 version_status; - __u16 minor_revision; - __u16 major_revision; - __u16 padding; - __u32 gpu_freq_khz_max; - __u32 log2_program_counter_size; - __u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; - __u8 paddings_1[4]; - __u64 gpu_available_memory_size; - __u8 num_exec_engines; - __u8 paddings_2[7]; -}; - -#endif /* _UAPI_BASE_JM_KERNEL_H_ */ diff --git a/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_ioctl.h b/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_ioctl.h deleted file mode 100644 index be554da15ba3..000000000000 --- a/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_ioctl.h +++ /dev/null @@ -1,274 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _UAPI_KBASE_JM_IOCTL_H_ -#define _UAPI_KBASE_JM_IOCTL_H_ - -#include -#include - -/* - * 11.1: - * - Add BASE_MEM_TILER_ALIGN_TOP under base_mem_alloc_flags - * 11.2: - * - KBASE_MEM_QUERY_FLAGS can return KBASE_REG_PF_GROW and KBASE_REG_PROTECTED, - * which some user-side clients prior to 11.2 might fault if they received - * them - * 11.3: - * - New ioctls KBASE_IOCTL_STICKY_RESOURCE_MAP and - * KBASE_IOCTL_STICKY_RESOURCE_UNMAP - * 11.4: - * - New ioctl KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET - * 11.5: - * - New ioctl: KBASE_IOCTL_MEM_JIT_INIT (old ioctl renamed to _OLD) - * 11.6: - * - Added flags field to base_jit_alloc_info structure, which can be used to - * specify pseudo chunked tiler alignment for JIT allocations. - * 11.7: - * - Removed UMP support - * 11.8: - * - Added BASE_MEM_UNCACHED_GPU under base_mem_alloc_flags - * 11.9: - * - Added BASE_MEM_PERMANENT_KERNEL_MAPPING and BASE_MEM_FLAGS_KERNEL_ONLY - * under base_mem_alloc_flags - * 11.10: - * - Enabled the use of nr_extres field of base_jd_atom_v2 structure for - * JIT_ALLOC and JIT_FREE type softjobs to enable multiple JIT allocations - * with one softjob. - * 11.11: - * - Added BASE_MEM_GPU_VA_SAME_4GB_PAGE under base_mem_alloc_flags - * 11.12: - * - Removed ioctl: KBASE_IOCTL_GET_PROFILING_CONTROLS - * 11.13: - * - New ioctl: KBASE_IOCTL_MEM_EXEC_INIT - * 11.14: - * - Add BASE_MEM_GROUP_ID_MASK, base_mem_group_id_get, base_mem_group_id_set - * under base_mem_alloc_flags - * 11.15: - * - Added BASEP_CONTEXT_MMU_GROUP_ID_MASK under base_context_create_flags. - * - Require KBASE_IOCTL_SET_FLAGS before BASE_MEM_MAP_TRACKING_HANDLE can be - * passed to mmap(). - * 11.16: - * - Extended ioctl KBASE_IOCTL_MEM_SYNC to accept imported dma-buf. - * - Modified (backwards compatible) ioctl KBASE_IOCTL_MEM_IMPORT behavior for - * dma-buf. Now, buffers are mapped on GPU when first imported, no longer - * requiring external resource or sticky resource tracking. UNLESS, - * CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND is enabled. - * 11.17: - * - Added BASE_JD_REQ_JOB_SLOT. - * - Reused padding field in base_jd_atom_v2 to pass job slot number. - * - New ioctl: KBASE_IOCTL_GET_CPU_GPU_TIMEINFO - * 11.18: - * - Added BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP under base_mem_alloc_flags - * 11.19: - * - Extended base_jd_atom_v2 to allow a renderpass ID to be specified. - * 11.20: - * - Added new phys_pages member to kbase_ioctl_mem_jit_init for - * KBASE_IOCTL_MEM_JIT_INIT, previous variants of this renamed to use _10_2 - * (replacing '_OLD') and _11_5 suffixes - * - Replaced compat_core_req (deprecated in 10.3) with jit_id[2] in - * base_jd_atom_v2. It must currently be initialized to zero. - * - Added heap_info_gpu_addr to base_jit_alloc_info, and - * BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE allowable in base_jit_alloc_info's - * flags member. Previous variants of this structure are kept and given _10_2 - * and _11_5 suffixes. - * - The above changes are checked for safe values in usual builds - * 11.21: - * - v2.0 of mali_trace debugfs file, which now versions the file separately - * 11.22: - * - Added base_jd_atom (v3), which is seq_nr + base_jd_atom_v2. - * KBASE_IOCTL_JOB_SUBMIT supports both in parallel. - * 11.23: - * - Modified KBASE_IOCTL_MEM_COMMIT behavior to reject requests to modify - * the physical memory backing of JIT allocations. This was not supposed - * to be a valid use case, but it was allowed by the previous implementation. - * 11.24: - * - Added a sysfs file 'serialize_jobs' inside a new sub-directory - * 'scheduling'. - * 11.25: - * - Enabled JIT pressure limit in base/kbase by default - * 11.26 - * - Added kinstr_jm API - * 11.27 - * - Backwards compatible extension to HWC ioctl. - * 11.28: - * - Added kernel side cache ops needed hint - * 11.29: - * - Reserve ioctl 52 - * 11.30: - * - Add a new priority level BASE_JD_PRIO_REALTIME - * - Add ioctl 54: This controls the priority setting. - * 11.31: - * - Added BASE_JD_REQ_LIMITED_CORE_MASK. - * - Added ioctl 55: set_limited_core_count. - * 11.32: - * - Added new HW performance counters interface to all GPUs. - * 11.33: - * - Removed Kernel legacy HWC interface - * 11.34: - * - First release of new HW performance counters interface. - * 11.35: - * - Dummy model (no mali) backend will now clear HWC values after each sample - * 11.36: - * - Remove legacy definitions: - * - base_jit_alloc_info_10_2 - * - base_jit_alloc_info_11_5 - * - kbase_ioctl_mem_jit_init_10_2 - * - kbase_ioctl_mem_jit_init_11_5 - * 11.37: - * - Fix kinstr_prfcnt issues: - * - Missing implicit sample for CMD_STOP when HWCNT buffer is full. - * - Race condition when stopping periodic sampling. - * - prfcnt_block_metadata::block_idx gaps. - * - PRFCNT_CONTROL_CMD_SAMPLE_ASYNC is removed. - * 11.38: - * - Relax the requirement to create a mapping with BASE_MEM_MAP_TRACKING_HANDLE - * before allocating GPU memory for the context. - * - CPU mappings of USER_BUFFER imported memory handles must be cached. - * 11.39: - * - Restrict child process from doing supported file operations (like mmap, ioctl, - * read, poll) on the file descriptor of mali device file that was inherited - * from the parent process. - * 11.40: - * - Remove KBASE_IOCTL_HWCNT_READER_SETUP and KBASE_HWCNT_READER_* ioctls. - * - Made the BASE_MEM_DONT_NEED memory flag queryable. - * 11.41: - * - Disallows changing the sharability on the GPU of imported dma-bufs to - * BASE_MEM_COHERENT_SYSTEM using KBASE_IOCTL_MEM_FLAGS_CHANGE. - * 11.42: - * - Implement full block state support for hardware counters. - * 11.43: - * - Made the BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP and BASE_MEM_KERNEL_SYNC memory - * flags queryable. - * 11.44: - * - Made the SAME_VA memory flag queryable. - * 11.45: - * - Re-allow child process to do supported file operations (like mmap, ioctl - * read, poll) on the file descriptor of mali device that was inherited - * from the parent process. - * 11.46: - * - Remove renderpass_id from base_jd_atom_v2 to deprecate support for JM Incremental Rendering - * 11.47: - * - Reject non-protected allocations containing the BASE_MEM_PROTECTED memory flag. - * - Reject allocations containing the BASE_MEM_DONT_NEED memory flag (it is only settable). - * - Reject allocations containing the BASE_MEM_UNUSED_BIT_xx memory flags. - * 11.48: - * - Add UNUSED_BIT_5, UNUSED_BIT_7, UNUSED_BIT_27 and UNUSED_BIT_29 previously occupied by - * kernel-only flags to kbase cap table. - * 11.49: - * - Increased KBASE_MEM_PROFILE_MAX_BUF_SIZE for more cctx memory classes. - */ - -#define BASE_UK_VERSION_MAJOR 11 -#define BASE_UK_VERSION_MINOR 49 - -/** - * struct kbase_ioctl_version_check - Check version compatibility between - * kernel and userspace - * - * @major: Major version number - * @minor: Minor version number - */ -struct kbase_ioctl_version_check { - __u16 major; - __u16 minor; -}; - -#define KBASE_IOCTL_VERSION_CHECK _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) - -/** - * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel - * - * @addr: Memory address of an array of struct base_jd_atom_v2 or v3 - * @nr_atoms: Number of entries in the array - * @stride: sizeof(struct base_jd_atom_v2) or sizeof(struct base_jd_atom) - */ -struct kbase_ioctl_job_submit { - __u64 addr; - __u32 nr_atoms; - __u32 stride; -}; - -#define KBASE_IOCTL_JOB_SUBMIT _IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit) - -#define KBASE_IOCTL_POST_TERM _IO(KBASE_IOCTL_TYPE, 4) - -/** - * struct kbase_ioctl_soft_event_update - Update the status of a soft-event - * @event: GPU address of the event which has been updated - * @new_status: The new status to set - * @flags: Flags for future expansion - */ -struct kbase_ioctl_soft_event_update { - __u64 event; - __u32 new_status; - __u32 flags; -}; - -#define KBASE_IOCTL_SOFT_EVENT_UPDATE \ - _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) - -/** - * struct kbase_kinstr_jm_fd_out - Explains the compatibility information for - * the `struct kbase_kinstr_jm_atom_state_change` structure returned from the - * kernel - * - * @size: The size of the `struct kbase_kinstr_jm_atom_state_change` - * @version: Represents a breaking change in the - * `struct kbase_kinstr_jm_atom_state_change` - * @padding: Explicit padding to get the structure up to 64bits. See - * https://www.kernel.org/doc/Documentation/ioctl/botching-up-ioctls.rst - * - * The `struct kbase_kinstr_jm_atom_state_change` may have extra members at the - * end of the structure that older user space might not understand. If the - * `version` is the same, the structure is still compatible with newer kernels. - * The `size` can be used to cast the opaque memory returned from the kernel. - */ -struct kbase_kinstr_jm_fd_out { - __u16 size; - __u8 version; - __u8 padding[5]; -}; - -/** - * struct kbase_kinstr_jm_fd_in - Options when creating the file descriptor - * - * @count: Number of atom states that can be stored in the kernel circular - * buffer. Must be a power of two - * @padding: Explicit padding to get the structure up to 64bits. See - * https://www.kernel.org/doc/Documentation/ioctl/botching-up-ioctls.rst - */ -struct kbase_kinstr_jm_fd_in { - __u16 count; - __u8 padding[6]; -}; - -union kbase_kinstr_jm_fd { - struct kbase_kinstr_jm_fd_in in; - struct kbase_kinstr_jm_fd_out out; -}; - -#define KBASE_IOCTL_KINSTR_JM_FD _IOWR(KBASE_IOCTL_TYPE, 51, union kbase_kinstr_jm_fd) - -#define KBASE_IOCTL_VERSION_CHECK_RESERVED \ - _IOWR(KBASE_IOCTL_TYPE, 52, struct kbase_ioctl_version_check) - -#endif /* _UAPI_KBASE_JM_IOCTL_H_ */ diff --git a/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h b/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h deleted file mode 100644 index 5818e01d1b31..000000000000 --- a/include/uapi/gpu/arm/valhall/jm/mali_kbase_jm_mem_flags.h +++ /dev/null @@ -1,65 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _UAPI_KBASE_JM_MEM_FLAGS_H_ -#define _UAPI_KBASE_JM_MEM_FLAGS_H_ - -/* Memory allocation, access/hint flags & mask specific to JM GPU. - * - * See base_mem_alloc_flags. - */ - -/* Unused bit for JM, only used in CSF for BASE_MEM_FIXED */ -#define BASE_MEM_UNUSED_BIT_8 ((base_mem_alloc_flags)1 << 8) - -/* Unused bit for JM, only used in CSF for BASE_CSF_EVENT */ -#define BASE_MEM_UNUSED_BIT_19 ((base_mem_alloc_flags)1 << 19) - -/** - * BASE_MEM_TILER_ALIGN_TOP - Memory starting from the end of the initial commit is aligned - * to 'extension' pages, where 'extension' must be a power of 2 and no more than - * BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES - */ -#define BASE_MEM_TILER_ALIGN_TOP ((base_mem_alloc_flags)1 << 20) - -/* Previously BASEP_MEM_PERFORM_JIT_TRIM, can be reused in the future */ -#define BASE_MEM_UNUSED_BIT_29 ((base_mem_alloc_flags)1 << 29) - -/* A mask of flags that, when provided, cause other flags to be - * enabled but are not enabled themselves - */ -#define BASE_MEM_FLAGS_ACTION_MODIFIERS (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) - -/* A mask of all currently reserved flags */ -#define BASE_MEM_FLAGS_RESERVED ((base_mem_alloc_flags)0) - -/* Number of bits used as flags for base memory management from kernel-side - * only (ie BASEP_MEM_* flags), located from 63 bit downwards: - * < 63 .. (64 - BASEP_MEM_FLAGS_NR_BITS) > - */ -#define BASEP_MEM_FLAGS_NR_BITS (6) - -/* A mask of all bits that are not used by a flag on JM */ -#define BASE_MEM_FLAGS_UNUSED \ - (BASE_MEM_UNUSED_BIT_5 | BASE_MEM_UNUSED_BIT_7 | BASE_MEM_UNUSED_BIT_8 | \ - BASE_MEM_UNUSED_BIT_19 | BASE_MEM_UNUSED_BIT_27 | BASE_MEM_UNUSED_BIT_29) - -#endif /* _UAPI_KBASE_JM_MEM_FLAGS_H_ */ diff --git a/include/uapi/gpu/arm/valhall/mali_base_common_kernel.h b/include/uapi/gpu/arm/valhall/mali_base_common_kernel.h index 66093c1689c1..ef09088b318c 100644 --- a/include/uapi/gpu/arm/valhall/mali_base_common_kernel.h +++ b/include/uapi/gpu/arm/valhall/mali_base_common_kernel.h @@ -25,12 +25,6 @@ #include #include "mali_kbase_mem_flags.h" -struct base_mem_handle { - struct { - __u64 handle; - } basep; -}; - #define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 4 /* Flags to pass to ::base_context_init. diff --git a/include/uapi/gpu/arm/valhall/mali_base_kernel.h b/include/uapi/gpu/arm/valhall/mali_base_kernel.h index 21e12a6f250a..2564860b0f12 100644 --- a/include/uapi/gpu/arm/valhall/mali_base_kernel.h +++ b/include/uapi/gpu/arm/valhall/mali_base_kernel.h @@ -167,26 +167,27 @@ struct base_fence { /** * struct base_mem_aliasing_info - Memory aliasing info * - * @handle: Handle to alias, can be BASE_MEM_WRITE_ALLOC_PAGES_HANDLE - * @offset: Offset within the handle to start aliasing from, in pages. - * Not used with BASE_MEM_WRITE_ALLOC_PAGES_HANDLE. - * @length: Length to alias, in pages. For BASE_MEM_WRITE_ALLOC_PAGES_HANDLE + * @gpu_va: GPU VA of an allocation to be aliased, can be a special value + * BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE + * @offset: Offset within the allocation to start aliasing from, in pages. + * Not used with BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE. + * @length: Length to alias, in pages. For BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE * specifies the number of times the special page is needed. * - * Describes a memory handle to be aliased. - * A subset of the handle can be chosen for aliasing, given an offset and a + * Describes a GPU memory allocation to be aliased. + * A subset of the allocation can be chosen for aliasing, given an offset and a * length. - * A special handle BASE_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a + * A special value BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE is used to represent a * region where a special page is mapped with a write-alloc cache setup, * typically used when the write result of the GPU isn't needed, but the GPU * must write anyway. * * Offset and length are specified in pages. - * Offset must be within the size of the handle. - * Offset+length must not overrun the size of the handle. + * Offset must be within the size of the allocation. + * Offset+length must not overrun the size of the allocation. */ struct base_mem_aliasing_info { - struct base_mem_handle handle; + __u64 gpu_va; __u64 offset; __u64 length; }; @@ -487,11 +488,7 @@ struct mali_base_gpu_coherent_group_info { struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS]; }; -#if MALI_USE_CSF #include "csf/mali_base_csf_kernel.h" -#else -#include "jm/mali_base_jm_kernel.h" -#endif /** * struct gpu_raw_gpu_props - A complete description of the GPU's Hardware diff --git a/include/uapi/gpu/arm/valhall/mali_base_mem_priv.h b/include/uapi/gpu/arm/valhall/mali_base_mem_priv.h index 1db3ff11d0c0..5ceda9b54b17 100644 --- a/include/uapi/gpu/arm/valhall/mali_base_mem_priv.h +++ b/include/uapi/gpu/arm/valhall/mali_base_mem_priv.h @@ -33,21 +33,21 @@ * It can either be: * @li a sync from CPU to Memory: * - type = ::BASE_SYNCSET_OP_MSYNC - * - mem_handle = a handle to the memory object on which the operation + * - gpu_va = GPU VA of GPU memory object on which the operation * is taking place * - user_addr = the address of the range to be synced * - size = the amount of data to be synced, in bytes * - offset is ignored. * @li a sync from Memory to CPU: * - type = ::BASE_SYNCSET_OP_CSYNC - * - mem_handle = a handle to the memory object on which the operation + * - gpu_va = GPU VA of GPU memory object on which the operation * is taking place * - user_addr = the address of the range to be synced * - size = the amount of data to be synced, in bytes. * - offset is ignored. */ struct basep_syncset { - struct base_mem_handle mem_handle; + __u64 gpu_va; __u64 user_addr; __u64 size; __u8 type; diff --git a/include/uapi/gpu/arm/valhall/mali_gpu_props.h b/include/uapi/gpu/arm/valhall/mali_gpu_props.h index 09ce542a76c4..5b4166017329 100644 --- a/include/uapi/gpu/arm/valhall/mali_gpu_props.h +++ b/include/uapi/gpu/arm/valhall/mali_gpu_props.h @@ -31,7 +31,9 @@ /** * struct gpu_props_user_data - structure for gpu props user buffer. * @core_props: Core props. + * @unused2: Extra space for address alignment. * @l2_props: L2 props. + * @unused4: Extra space for address alignment. * @tiler_props: Tiler props. * @thread_props: Thread props. * @raw_props: Raw register values kept for backwards compatibility. Kbase @@ -47,17 +49,21 @@ struct gpu_props_user_data { __u16 version_status; __u16 minor_revision; __u16 major_revision; + __u8 unused0[2]; __u32 gpu_freq_khz_max; __u32 log2_program_counter_size; __u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; + __u8 unused1[4]; __u64 gpu_available_memory_size; __u8 num_exec_engines; } core_props; + __u8 unused2[7]; struct { __u8 log2_line_size; __u8 log2_cache_size; __u8 num_l2_slices; } l2_props; + __u8 unused4[2]; struct { __u32 bin_size_bytes; __u32 max_active_levels; @@ -70,6 +76,7 @@ struct gpu_props_user_data { __u8 max_task_queue; __u8 max_thread_group_split; __u8 impl_tech; + __u8 unused5; __u32 tls_alloc; } thread_props; @@ -103,6 +110,7 @@ struct gpu_props_user_data { __u32 num_groups; __u32 num_core_groups; __u32 coherency; + __u8 unused6[4]; struct { __u64 core_mask; __u32 num_cores; diff --git a/include/uapi/gpu/arm/valhall/mali_kbase_ioctl.h b/include/uapi/gpu/arm/valhall/mali_kbase_ioctl.h index 6c3f8f127fd9..46ea47f252b6 100644 --- a/include/uapi/gpu/arm/valhall/mali_kbase_ioctl.h +++ b/include/uapi/gpu/arm/valhall/mali_kbase_ioctl.h @@ -29,11 +29,7 @@ extern "C" { #include #include -#if MALI_USE_CSF #include "csf/mali_kbase_csf_ioctl.h" -#else -#include "jm/mali_kbase_jm_ioctl.h" -#endif /* MALI_USE_CSF */ #define KBASE_IOCTL_TYPE 0x80 @@ -236,9 +232,9 @@ struct kbase_ioctl_mem_jit_init { #define KBASE_IOCTL_MEM_JIT_INIT _IOW(KBASE_IOCTL_TYPE, 14, struct kbase_ioctl_mem_jit_init) /** - * struct kbase_ioctl_mem_sync - Perform cache maintenance on memory + * struct kbase_ioctl_mem_sync - Perform CPU cache maintenance on GPU memory * - * @handle: GPU memory handle (GPU VA) + * @gpu_va: GPU VA of the memory for which cache maintenance needs to be performed. * @user_addr: The address where it is mapped in user space * @size: The number of bytes to synchronise * @type: The direction to synchronise: 0 is sync to memory (clean), @@ -246,7 +242,7 @@ struct kbase_ioctl_mem_jit_init { * @padding: Padding to round up to a multiple of 8 bytes, must be zero */ struct kbase_ioctl_mem_sync { - __u64 handle; + __u64 gpu_va; __u64 user_addr; __u64 size; __u8 type; diff --git a/include/uapi/gpu/arm/valhall/mali_kbase_kinstr_jm_reader.h b/include/uapi/gpu/arm/valhall/mali_kbase_kinstr_jm_reader.h deleted file mode 100644 index 72e1b9dbf245..000000000000 --- a/include/uapi/gpu/arm/valhall/mali_kbase_kinstr_jm_reader.h +++ /dev/null @@ -1,69 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* - * mali_kbase_kinstr_jm_reader.h - * Provides an ioctl API to read kernel atom state changes. The flow of the - * API is: - * 1. Obtain the file descriptor with ``KBASE_IOCTL_KINSTR_JM_FD`` - * 2. Determine the buffer structure layout via the above ioctl's returned - * size and version fields in ``struct kbase_kinstr_jm_fd_out`` - * 4. Poll the file descriptor for ``POLLIN`` - * 5. Get data with read() on the fd - * 6. Use the structure version to understand how to read the data from the - * buffer - * 7. Repeat 4-6 - * 8. Close the file descriptor - */ - -#ifndef _UAPI_KBASE_KINSTR_JM_READER_H_ -#define _UAPI_KBASE_KINSTR_JM_READER_H_ - -/** - * enum kbase_kinstr_jm_reader_atom_state - Determines the work state of an atom - * @KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE: Signifies that an atom has - * entered a hardware queue - * @KBASE_KINSTR_JM_READER_ATOM_STATE_START: Signifies that work has started - * on an atom - * @KBASE_KINSTR_JM_READER_ATOM_STATE_STOP: Signifies that work has stopped - * on an atom - * @KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE: Signifies that work has - * completed on an atom - * @KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT: The number of state enumerations - * - * We can add new states to the end of this if they do not break the existing - * state machine. Old user mode code can gracefully ignore states they do not - * understand. - * - * If we need to make a breaking change to the state machine, we can do that by - * changing the version reported by KBASE_IOCTL_KINSTR_JM_FD. This will - * mean that old user mode code will fail to understand the new state field in - * the structure and gracefully not use the state change API. - */ -enum kbase_kinstr_jm_reader_atom_state { - KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE, - KBASE_KINSTR_JM_READER_ATOM_STATE_START, - KBASE_KINSTR_JM_READER_ATOM_STATE_STOP, - KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE, - KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT -}; - -#endif /* _UAPI_KBASE_KINSTR_JM_READER_H_ */ diff --git a/include/uapi/gpu/arm/valhall/mali_kbase_mem_flags.h b/include/uapi/gpu/arm/valhall/mali_kbase_mem_flags.h index 465d27a014c2..04d5ca3c3d8a 100644 --- a/include/uapi/gpu/arm/valhall/mali_kbase_mem_flags.h +++ b/include/uapi/gpu/arm/valhall/mali_kbase_mem_flags.h @@ -22,11 +22,7 @@ #ifndef _UAPI_KBASE_MEM_FLAGS_H_ #define _UAPI_KBASE_MEM_FLAGS_H_ -#if MALI_USE_CSF #include "csf/mali_kbase_csf_mem_flags.h" -#else -#include "jm/mali_kbase_jm_mem_flags.h" -#endif /* Memory allocation, access/hint flags & mask. * From 2e1afa3ce5fffb807c081df375add359257c0ce1 Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Sat, 12 Apr 2025 16:52:56 +0800 Subject: [PATCH 45/49] Mali: valhall: from ARM: Fix UAF issue of user IO pages after group fatal error The user IO page mappings were not unmapped properly when a group is terminated, which could lead to the pages being accessed after termination. With this change, the driver shall remap a group's shared page to the dummy page during termination to ensure that accesses to it will not cause the application to crash, but also these accesses will be rendered harmless. For RK, this modification is derived from the kernel_patch_from_Eric_250412 provided by ARM in support case 03504455. It is originally intended to address the issue that the GKI of kernel 5.10 does not provide 'zap_vma_ptes' which the r54 driver depends on. Change-Id: I28a6946436343d2b44fe7e4df9b4bb8bbd3bafe1 Signed-off-by: Zhen Chen --- drivers/gpu/arm/valhall/csf/mali_kbase_csf.c | 12 ++++++----- .../gpu/arm/valhall/csf/mali_kbase_csf_defs.h | 13 ------------ .../gpu/arm/valhall/mali_kbase_mem_linux.c | 21 ++++++++++++------- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c b/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c index 26ebe40690cc..22214f5b95e7 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf.c @@ -211,11 +211,6 @@ static void kernel_free_user_io_pages(struct kbase_context *kctx, struct tagged_ kbase_gpu_vm_lock(kctx); vunmap(user_io_addr); - if (kctx->csf.user_io.vma != NULL) { - zap_vma_ptes(kctx->csf.user_io.vma, kctx->csf.user_io.vma->vm_start, - KBASEP_NUM_CS_USER_IO_PAGES * PAGE_SIZE); - kctx->csf.user_io.vma = NULL; - } WARN_ON(atomic_read(&kctx->permanent_mapped_pages) < KBASEP_NUM_CS_USER_IO_PAGES); atomic_sub(KBASEP_NUM_CS_USER_IO_PAGES, &kctx->permanent_mapped_pages); @@ -1028,6 +1023,13 @@ static void unbind_stopped_queue(struct kbase_context *kctx, struct kbase_queue queue->group = NULL; kbase_csf_scheduler_spin_unlock(kctx->kbdev, flags); + /* Ensure that the user I/O pages are no longer accessible */ + mutex_lock(&kctx->kbdev->csf.reg_lock); + unmap_mapping_range(kctx->kbdev->csf.db_filp->f_inode->i_mapping, + (loff_t)(queue->db_file_offset << PAGE_SHIFT), + BASEP_QUEUE_NR_MMAP_USER_PAGES * PAGE_SIZE, 1); + mutex_unlock(&kctx->kbdev->csf.reg_lock); + put_user_pages_mmap_handle(kctx, queue); WARN_ON_ONCE(queue->doorbell_nr != KBASEP_USER_DB_NR_INVALID); queue->bind_state = KBASE_CSF_QUEUE_UNBOUND; diff --git a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h index 16741079cb36..48569836e320 100644 --- a/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h +++ b/drivers/gpu/arm/valhall/csf/mali_kbase_csf_defs.h @@ -927,17 +927,6 @@ struct kbase_csf_event { spinlock_t lock; }; -/** - * struct kbase_csf_user_io_context - Object containing members to manage the mapping - * of USER io page for a context. - * - * @vma: Pointer to the VMA corresponding to the virtual mapping - * of the USER io page. - */ -struct kbase_csf_user_io_context { - struct vm_area_struct *vma; -}; - /** * struct kbase_csf_user_reg_context - Object containing members to manage the mapping * of USER Register page for a context. @@ -997,7 +986,6 @@ struct kbase_csf_user_reg_context { * @cpu_queue: CPU queue information. Only be available when DEBUG_FS * is enabled. * @user_reg: Collective information to support mapping to USER Register page. - * @user_io: Collective information to support mapping to USER IO page. * @pending_sync_update: Indicates that kbase_csf_scheduler_kthread() should * handle SYNC_UPDATE event for this context. This would * be set to false when the work is done. This is used @@ -1018,7 +1006,6 @@ struct kbase_csf_context { struct kbase_csf_scheduler_context sched; struct kbase_csf_cpu_queue_context cpu_queue; struct kbase_csf_user_reg_context user_reg; - struct kbase_csf_user_io_context user_io; atomic_t pending_sync_update; }; diff --git a/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c b/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c index d7301653788e..c8e36707aae9 100644 --- a/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c +++ b/drivers/gpu/arm/valhall/mali_kbase_mem_linux.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2025 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -3440,13 +3440,19 @@ static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_fault *vmf) output_cpu_addr = input_cpu_addr + PAGE_SIZE; if (mali_kbase_supports_csg_cs_user_page_allocation(queue->kctx->api_version)) { - if (!queue->group) { - ret = VM_FAULT_SIGBUS; - goto exit; + if (likely(queue->group)) { + input_page_pfn = PFN_DOWN(as_phys_addr_t(queue->group->phys[0])); + output_page_pfn = PFN_DOWN(as_phys_addr_t(queue->group->phys[1])); + } else { + /* This could happen if userspace tries to access this memory + * after the group has already been terminated due to a fault. + * Re-map to the dummy page to render the access harmless. + */ + input_page_pfn = + PFN_DOWN(as_phys_addr_t(kbdev->csf.user_reg.dummy_page)); + output_page_pfn = + PFN_DOWN(as_phys_addr_t(kbdev->csf.user_reg.dummy_page)); } - - input_page_pfn = PFN_DOWN(as_phys_addr_t(queue->group->phys[0])); - output_page_pfn = PFN_DOWN(as_phys_addr_t(queue->group->phys[1])); } else { input_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[0])); output_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[1])); @@ -3541,7 +3547,6 @@ static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, struct v get_file(vma->vm_file); /* Also adjust the vm_pgoff */ vma->vm_pgoff = queue->db_file_offset; - kctx->csf.user_io.vma = vma; return 0; From 82a3cc4aedd0162f4374eaed9b44182f95366b2d Mon Sep 17 00:00:00 2001 From: Zhibin Huang Date: Mon, 26 May 2025 10:45:24 +0800 Subject: [PATCH 46/49] input: touchscreen: gt1x: return error when initialization fails Type: Fix Redmine ID: #N/A Associated modifications: N/A Test: N/A Signed-off-by: Zhibin Huang Change-Id: Ib6d46df548e0448855e6088bc2e8fa2f7df2d352 --- drivers/input/touchscreen/gt1x/gt1x_generic.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/input/touchscreen/gt1x/gt1x_generic.c b/drivers/input/touchscreen/gt1x/gt1x_generic.c index 3840444c0131..abbf764b3fd1 100644 --- a/drivers/input/touchscreen/gt1x/gt1x_generic.c +++ b/drivers/input/touchscreen/gt1x/gt1x_generic.c @@ -2354,18 +2354,21 @@ s32 gt1x_init(void) ret = gt1x_get_chip_type(); if (ret != 0) { GTP_ERROR("Get chip type failed!"); + goto init_err; } /* read version information */ ret = gt1x_read_version(>1x_version); if (ret != 0) { GTP_ERROR("Get verision failed!"); + goto init_err; } /* init and send configs */ ret = gt1x_init_panel(); if (ret != 0) { GTP_ERROR("Init panel failed."); + goto init_err; } gt1x_workqueue = create_singlethread_workqueue("gt1x_workthread"); @@ -2403,6 +2406,8 @@ s32 gt1x_init(void) #if GTP_WITH_STYLUS gt1x_pen_init(); #endif + +init_err: if (ret != 0) gt1x_power_switch(SWITCH_OFF); From 8ea643bc7c39ccfaaac046e4fea09d58162aba43 Mon Sep 17 00:00:00 2001 From: Luo Wei Date: Tue, 10 Jun 2025 18:25:45 +0800 Subject: [PATCH 47/49] arm64: configs: vehicle.config: enable vehicle driver default Signed-off-by: Luo Wei Change-Id: I28878e58eab57fd5cc6e23bfd9bf09adceed8791 --- arch/arm64/configs/rk3576_vehicle.config | 44 ++++++++++++++++++++---- arch/arm64/configs/rk3588_vehicle.config | 43 +++++++++++++++++++---- 2 files changed, 75 insertions(+), 12 deletions(-) diff --git a/arch/arm64/configs/rk3576_vehicle.config b/arch/arm64/configs/rk3576_vehicle.config index 0df4caf1b138..e58d71d0f0f2 100644 --- a/arch/arm64/configs/rk3576_vehicle.config +++ b/arch/arm64/configs/rk3576_vehicle.config @@ -39,6 +39,7 @@ CONFIG_HZ_1000=y # CONFIG_LIGHT_DEVICE is not set CONFIG_LOG_BUF_SHIFT=20 # CONFIG_MALI400 is not set +CONFIG_MALI_BIFROST=y # CONFIG_MALI_MIDGARD is not set # CONFIG_MFD_MAX96745 is not set # CONFIG_MFD_MAX96755F is not set @@ -57,8 +58,6 @@ CONFIG_REALTEK_PHY=y # CONFIG_REGULATOR_WL2868C is not set # CONFIG_REGULATOR_XZ3216 is not set # CONFIG_ROCKCHIP_CHARGER_MANAGER is not set -# CONFIG_ROCKCHIP_CLK_INV is not set -# CONFIG_ROCKCHIP_CLK_PVTM is not set CONFIG_ROCKCHIP_DRM_DIRECT_SHOW=y CONFIG_RTC_DRV_S35390A=y # CONFIG_SLUB_SYSFS is not set @@ -73,6 +72,7 @@ CONFIG_RTC_DRV_S35390A=y # CONFIG_SND_SOC_RK817 is not set # CONFIG_SND_SOC_RK_CODEC_DIGITAL is not set # CONFIG_SND_SOC_RT5640 is not set +CONFIG_SPI_SLAVE=y # CONFIG_TOUCHSCREEN_ELAN5515 is not set # CONFIG_TOUCHSCREEN_GSL3673 is not set # CONFIG_TOUCHSCREEN_GSLX680_PAD is not set @@ -93,6 +93,7 @@ CONFIG_TOUCHSCREEN_ILI210X=m # CONFIG_USB_NET_SMSC95XX is not set CONFIG_USB_OHCI_HCD=m CONFIG_USB_OHCI_HCD_PLATFORM=m +CONFIG_VEHICLE_CORE=y # CONFIG_VIDEO_AW36518 is not set # CONFIG_VIDEO_AW8601 is not set # CONFIG_VIDEO_CN3927V is not set @@ -117,6 +118,32 @@ CONFIG_VIDEO_MAXIM_SERDES=y # CONFIG_VIDEO_S5KJN1 is not set # CONFIG_VIDEO_SGM3784 is not set # CONFIG_VL6180 is not set +CONFIG_LARGE_PAGE_SUPPORT=y +# CONFIG_MALI_ARBITER_SUPPORT is not set +# CONFIG_MALI_ARBITRATION is not set +CONFIG_MALI_BIFROST_DEBUG=y +CONFIG_MALI_BIFROST_DEVFREQ=y +CONFIG_MALI_BIFROST_ENABLE_TRACE=y +CONFIG_MALI_BIFROST_EXPERT=y +CONFIG_MALI_BIFROST_FENCE_DEBUG=y +CONFIG_MALI_BIFROST_GATOR_SUPPORT=y +# CONFIG_MALI_BIFROST_NO_MALI is not set +# CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY is not set +CONFIG_MALI_BIFROST_SYSTEM_TRACE=y +# CONFIG_MALI_CSF_SUPPORT is not set +# CONFIG_MALI_DMA_BUF_LEGACY_COMPAT is not set +# CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND is not set +# CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED is not set +# CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE is not set +# CONFIG_MALI_IS_FPGA is not set +# CONFIG_MALI_JOB_DUMP is not set +CONFIG_MALI_PLATFORM_NAME="rk" +CONFIG_MALI_PRFCNT_SET_PRIMARY=y +# CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS is not set +# CONFIG_MALI_PRFCNT_SET_TERTIARY is not set +CONFIG_MALI_REAL_HW=y +CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD=y +CONFIG_PAGE_MIGRATION_SUPPORT=y # CONFIG_ROCKCHIP_DRM_SELF_TEST is not set CONFIG_SERDES_DISPLAY_CHIP_MAXIM=y CONFIG_SERDES_DISPLAY_CHIP_MAXIM_MAX96745=y @@ -132,6 +159,15 @@ CONFIG_SERDES_DISPLAY_CHIP_ROCKCHIP_RKX121=y CONFIG_SERDES_DISPLAY_CHIP_ROHM=y CONFIG_SERDES_DISPLAY_CHIP_ROHM_BU18RL82=y CONFIG_SERDES_DISPLAY_CHIP_ROHM_BU18TL82=y +CONFIG_SPI_DYNAMIC=y +# CONFIG_SPI_SLAVE_ROCKCHIP_OBJ is not set +# CONFIG_SPI_SLAVE_SYSTEM_CONTROL is not set +# CONFIG_SPI_SLAVE_TIME is not set +CONFIG_VEHICLE_ADC=y +CONFIG_VEHICLE_DUMMY=y +CONFIG_VEHICLE_GPIO=y +CONFIG_VEHICLE_SPI=y +# CONFIG_VEHICLE_SPI_PROTOCOL is not set CONFIG_VIDEO_MAXIM_CAM_DUMMY=y # CONFIG_VIDEO_MAXIM_CAM_OS04A10 is not set CONFIG_VIDEO_MAXIM_CAM_OV231X=y @@ -144,7 +180,3 @@ CONFIG_VIDEO_MAXIM_SER_MAX9295=y CONFIG_VIDEO_MAXIM_SER_MAX96715=y CONFIG_VIDEO_MAXIM_SER_MAX96717=y # CONFIG_VIDEO_REVERSE_IMAGE is not set -CONFIG_MALI_BIFROST=y -CONFIG_MALI_PLATFORM_NAME="rk" -CONFIG_MALI_BIFROST_EXPERT=y -CONFIG_MALI_BIFROST_DEBUG=y diff --git a/arch/arm64/configs/rk3588_vehicle.config b/arch/arm64/configs/rk3588_vehicle.config index 722de0be5abf..1bf6dca4d8d8 100644 --- a/arch/arm64/configs/rk3588_vehicle.config +++ b/arch/arm64/configs/rk3588_vehicle.config @@ -40,6 +40,7 @@ CONFIG_HZ_1000=y CONFIG_LOG_BUF_SHIFT=20 # CONFIG_MALI400 is not set # CONFIG_MALI_MIDGARD is not set +CONFIG_MALI_VALHALL=y # CONFIG_MFD_MAX96745 is not set # CONFIG_MFD_MAX96755F is not set # CONFIG_MFD_RK618 is not set @@ -57,8 +58,6 @@ CONFIG_REALTEK_PHY=y # CONFIG_REGULATOR_WL2868C is not set # CONFIG_REGULATOR_XZ3216 is not set # CONFIG_ROCKCHIP_CHARGER_MANAGER is not set -# CONFIG_ROCKCHIP_CLK_INV is not set -# CONFIG_ROCKCHIP_CLK_PVTM is not set CONFIG_ROCKCHIP_DRM_DIRECT_SHOW=y CONFIG_RTC_DRV_S35390A=y # CONFIG_SLUB_SYSFS is not set @@ -73,6 +72,7 @@ CONFIG_RTC_DRV_S35390A=y # CONFIG_SND_SOC_RK817 is not set # CONFIG_SND_SOC_RK_CODEC_DIGITAL is not set # CONFIG_SND_SOC_RT5640 is not set +CONFIG_SPI_SLAVE=y # CONFIG_TOUCHSCREEN_ELAN5515 is not set # CONFIG_TOUCHSCREEN_GSL3673 is not set # CONFIG_TOUCHSCREEN_GSLX680_PAD is not set @@ -93,6 +93,7 @@ CONFIG_TOUCHSCREEN_ILI210X=m # CONFIG_USB_NET_SMSC95XX is not set CONFIG_USB_OHCI_HCD=m CONFIG_USB_OHCI_HCD_PLATFORM=m +CONFIG_VEHICLE_CORE=y # CONFIG_VIDEO_AW36518 is not set # CONFIG_VIDEO_AW8601 is not set # CONFIG_VIDEO_CN3927V is not set @@ -117,6 +118,30 @@ CONFIG_VIDEO_MAXIM_SERDES=y # CONFIG_VIDEO_S5KJN1 is not set # CONFIG_VIDEO_SGM3784 is not set # CONFIG_VL6180 is not set +CONFIG_MALI_CSF_INCLUDE_FW=y +# CONFIG_MALI_VALHALL_ARBITRATION is not set +# CONFIG_MALI_VALHALL_CORESIGHT is not set +# CONFIG_MALI_VALHALL_CORESTACK is not set +CONFIG_MALI_VALHALL_CSF_SUPPORT=y +CONFIG_MALI_VALHALL_DEBUG=y +CONFIG_MALI_VALHALL_DEVFREQ=y +# CONFIG_MALI_VALHALL_DMA_BUF_LEGACY_COMPAT is not set +# CONFIG_MALI_VALHALL_DMA_BUF_MAP_ON_DEMAND is not set +CONFIG_MALI_VALHALL_ENABLE_TRACE=y +CONFIG_MALI_VALHALL_EXPERT=y +CONFIG_MALI_VALHALL_FENCE_DEBUG=y +CONFIG_MALI_VALHALL_GATOR_SUPPORT=y +# CONFIG_MALI_VALHALL_HW_ERRATA_1485982_NOT_AFFECTED is not set +# CONFIG_MALI_VALHALL_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE is not set +# CONFIG_MALI_VALHALL_JOB_DUMP is not set +# CONFIG_MALI_VALHALL_NO_MALI is not set +CONFIG_MALI_VALHALL_PLATFORM_NAME="rk" +CONFIG_MALI_VALHALL_PRFCNT_SET_PRIMARY=y +# CONFIG_MALI_VALHALL_PRFCNT_SET_SECONDARY is not set +# CONFIG_MALI_VALHALL_PRFCNT_SET_TERTIARY is not set +CONFIG_MALI_VALHALL_REAL_HW=y +CONFIG_MALI_VALHALL_SYSTEM_TRACE=y +CONFIG_MALI_VALHALL_TRACE_POWER_GPU_WORK_PERIOD=y # CONFIG_ROCKCHIP_DRM_SELF_TEST is not set CONFIG_SERDES_DISPLAY_CHIP_MAXIM=y CONFIG_SERDES_DISPLAY_CHIP_MAXIM_MAX96745=y @@ -132,6 +157,16 @@ CONFIG_SERDES_DISPLAY_CHIP_ROCKCHIP_RKX121=y CONFIG_SERDES_DISPLAY_CHIP_ROHM=y CONFIG_SERDES_DISPLAY_CHIP_ROHM_BU18RL82=y CONFIG_SERDES_DISPLAY_CHIP_ROHM_BU18TL82=y +CONFIG_SPI_DYNAMIC=y +# CONFIG_SPI_SLAVE_ROCKCHIP_OBJ is not set +# CONFIG_SPI_SLAVE_SYSTEM_CONTROL is not set +# CONFIG_SPI_SLAVE_TIME is not set +CONFIG_VALHALL_LARGE_PAGE_SUPPORT=y +CONFIG_VEHICLE_ADC=y +CONFIG_VEHICLE_DUMMY=y +CONFIG_VEHICLE_GPIO=y +CONFIG_VEHICLE_SPI=y +# CONFIG_VEHICLE_SPI_PROTOCOL is not set CONFIG_VIDEO_MAXIM_CAM_DUMMY=y CONFIG_VIDEO_MAXIM_CAM_OS04A10=y CONFIG_VIDEO_MAXIM_CAM_OV231X=y @@ -144,7 +179,3 @@ CONFIG_VIDEO_MAXIM_SER_MAX9295=y CONFIG_VIDEO_MAXIM_SER_MAX96715=y CONFIG_VIDEO_MAXIM_SER_MAX96717=y # CONFIG_VIDEO_REVERSE_IMAGE is not set -CONFIG_MALI_VALHALL=y -CONFIG_MALI_VALHALL_PLATFORM_NAME="rk" -CONFIG_MALI_VALHALL_EXPERT=y -CONFIG_MALI_VALHALL_DEBUG=y From fe59c73bbf82192e5f1a0de5d3dcd61727461fe5 Mon Sep 17 00:00:00 2001 From: Weixin Zhou Date: Mon, 16 Jun 2025 17:46:11 +0800 Subject: [PATCH 48/49] MALI: bifrost: Optimize gpu mem sysfs entry One process corresponds to only one node, simplifying the upper-level access. ls -l sys/class/misc/mali0/device/kprcs/(pid)/ -r--r--r-- 1 root root 4096 2025-06-16 09:47 private_gpu_mem -r--r--r-- 1 root root 4096 2025-06-16 09:47 total_gpu_mem Signed-off-by: Weixin Zhou Change-Id: Ia198059560c84c5097bfb04f9a16c056db1edaf3 --- .../arm/bifrost/context/mali_kbase_context.c | 81 +++++++++++++++++++ .../gpu/arm/bifrost/mali_kbase_core_linux.c | 72 +---------------- 2 files changed, 83 insertions(+), 70 deletions(-) diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context.c b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c index 2c7417bd6506..b9a6c2485de8 100644 --- a/drivers/gpu/arm/bifrost/context/mali_kbase_context.c +++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c @@ -72,6 +72,75 @@ static struct kbase_process *find_process_node(struct rb_node *node, pid_t tgid) return kprcs; } +static ssize_t kbase_kctx_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct kobj_attribute *kattr = container_of(attr, struct kobj_attribute, attr); + + if (kattr->show) + return kattr->show(kobj, kattr, buf); + + return -EIO; +} + +static ssize_t kbase_total_gpu_mem_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct kbase_context *kctx = container_of(kobj, struct kbase_context, kobj); + + return scnprintf(buf, PAGE_SIZE, "%zu\n", kctx->kprcs->total_gpu_pages << PAGE_SHIFT); +} + +static ssize_t kbase_private_gpu_mem_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct kbase_context *kctx = container_of(kobj, struct kbase_context, kobj); + struct kbase_process *kprcs = kctx->kprcs; + struct kbase_context *tmp_kctx; + size_t total_pages = 0; + + /* Sum up used_pages from all contexts in the process */ + list_for_each_entry(tmp_kctx, &kprcs->kctx_list, kprcs_link) { + total_pages += atomic_read(&tmp_kctx->used_pages); + } + + return scnprintf(buf, PAGE_SIZE, "%zu\n", total_pages << PAGE_SHIFT); +} + +static struct kobj_attribute kbase_total_gpu_mem_attr = { + .attr = { + .name = "total_gpu_mem", + .mode = 0444, + }, + .show = kbase_total_gpu_mem_show, + .store = NULL, +}; + +static struct kobj_attribute kbase_private_gpu_mem_attr = { + .attr = { + .name = "private_gpu_mem", + .mode = 0444, + }, + .show = kbase_private_gpu_mem_show, + .store = NULL, +}; + +static struct attribute *kbase_kctx_attrs[] = { + &kbase_total_gpu_mem_attr.attr, + &kbase_private_gpu_mem_attr.attr, + NULL, +}; + +static const struct attribute_group kbase_kctx_attr_group = { + .attrs = kbase_kctx_attrs, +}; + +static const struct sysfs_ops kbase_kctx_sysfs_ops = { + .show = kbase_kctx_attr_show, +}; + +static const struct kobj_type kbase_kctx_ktype = { + .sysfs_ops = &kbase_kctx_sysfs_ops, + .default_groups = (const struct attribute_group *[]) { &kbase_kctx_attr_group, NULL }, +}; + /** * kbase_insert_kctx_to_process - Initialise kbase process context. * @@ -100,6 +169,8 @@ static int kbase_insert_kctx_to_process(struct kbase_context *kctx) */ if (!kprcs) { struct rb_node **new = &prcs_root->rb_node, *parent = NULL; + char kctx_name[64]; + int ret = 0; kprcs = kzalloc(sizeof(*kprcs), GFP_KERNEL); if (kprcs == NULL) @@ -109,6 +180,15 @@ static int kbase_insert_kctx_to_process(struct kbase_context *kctx) kprcs->dma_buf_root = RB_ROOT; kprcs->total_gpu_pages = 0; + if (unlikely(!scnprintf(kctx_name, 64, "%d", tgid))) + return -ENOMEM; + + ret = kobject_init_and_add(&kctx->kobj, &kbase_kctx_ktype, kctx->kbdev->kprcs_kobj, kctx_name); + if (ret) { + dev_err(kctx->kbdev->dev, "Failed to create kctx kobject"); + kobject_put(&kctx->kobj); + } + while (*new) { struct kbase_process *prcs_node; @@ -260,6 +340,7 @@ static void kbase_remove_kctx_from_process(struct kbase_context *kctx) * we can remove it from the process rb_tree. */ if (list_empty(&kprcs->kctx_list)) { + kobject_put(&kctx->kobj); rb_erase(&kprcs->kprcs_node, &kctx->kbdev->process_root); /* Add checks, so that the terminating process Should not * hold any gpu_memory. diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c index 0f41b0411faa..46c91c6fcb6c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c @@ -360,8 +360,6 @@ static void kbase_file_delete(struct kbase_file *const kfile) #endif kbase_context_debugfs_term(kctx); - kobject_put(&kctx->kobj); - kbase_destroy_context(kctx); dev_dbg(kbdev->dev, "deleted base context\n"); @@ -626,74 +624,14 @@ static const struct file_operations kbase_force_same_va_fops = { }; #endif /* CONFIG_DEBUG_FS */ -static ssize_t kbase_kctx_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) -{ - struct kobj_attribute *kattr = container_of(attr, struct kobj_attribute, attr); - - if (kattr->show) - return kattr->show(kobj, kattr, buf); - - return -EIO; -} - -static ssize_t kbase_total_gpu_mem_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) -{ - struct kbase_context *kctx = container_of(kobj, struct kbase_context, kobj); - - return scnprintf(buf, PAGE_SIZE, "%zu\n", kctx->kprcs->total_gpu_pages << PAGE_SHIFT); -} - -static ssize_t kbase_private_gpu_mem_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) -{ - struct kbase_context *kctx = container_of(kobj, struct kbase_context, kobj); - - return scnprintf(buf, PAGE_SIZE, "%d\n", atomic_read(&kctx->used_pages) << PAGE_SHIFT); -} - -static struct kobj_attribute kbase_total_gpu_mem_attr = { - .attr = { - .name = "total_gpu_mem", - .mode = 0444, - }, - .show = kbase_total_gpu_mem_show, - .store = NULL, -}; - -static struct kobj_attribute kbase_private_gpu_mem_attr = { - .attr = { - .name = "private_gpu_mem", - .mode = 0444, - }, - .show = kbase_private_gpu_mem_show, - .store = NULL, -}; - -static struct attribute *kbase_kctx_attrs[] = { - &kbase_total_gpu_mem_attr.attr, - &kbase_private_gpu_mem_attr.attr, - NULL, -}; - -static const struct attribute_group kbase_kctx_attr_group = { - .attrs = kbase_kctx_attrs, -}; - -static const struct sysfs_ops kbase_kctx_sysfs_ops = { - .show = kbase_kctx_attr_show, -}; - -static const struct kobj_type kbase_kctx_ktype = { - .sysfs_ops = &kbase_kctx_sysfs_ops, - .default_groups = (const struct attribute_group *[]) { &kbase_kctx_attr_group, NULL }, -}; - static int kbase_file_create_kctx(struct kbase_file *const kfile, base_context_create_flags const flags) { struct kbase_device *kbdev = NULL; struct kbase_context *kctx = NULL; +#if IS_ENABLED(CONFIG_DEBUG_FS) char kctx_name[64]; - int ret = 0; +#endif if (WARN_ON(!kfile)) return -EINVAL; @@ -738,12 +676,6 @@ static int kbase_file_create_kctx(struct kbase_file *const kfile, } #endif /* CONFIG_DEBUG_FS */ - ret = kobject_init_and_add(&kctx->kobj, &kbase_kctx_ktype, kbdev->kprcs_kobj, kctx_name); - if (ret) { - dev_err(kbdev->dev, "Failed to create kctx kobject"); - kobject_put(&kctx->kobj); - } - dev_dbg(kbdev->dev, "created base context\n"); kfile->kctx = kctx; From 66cc998fcfe01db4376f8706b6dc5825030cd372 Mon Sep 17 00:00:00 2001 From: Andy Yan Date: Tue, 17 Jun 2025 17:46:08 +0800 Subject: [PATCH 49/49] drm/rockchip: vop2: Fix vcstate for splice vp The vcstate for the splice vp may be NULL if this vp not registered as a crtc, this may trigger a null pointer access in the 8K mode: [ 84.267856] Unable to handle kernel NULL pointer dereference at virtual address 0000000000000040 [ 84.344984] Mem abort info: [ 84.347778] ESR = 0x0000000096000005 [ 84.392721] Internal error: Oops: 0000000096000005 [#1] SMP [ 84.398289] Modules linked in: [ 84.401347] CPU: 6 PID: 779 Comm: weston Not tainted 6.1.99 #21 [ 84.407263] Hardware name: IP11HH-8K-104 V0.4 (DT) [ 84.412048] pstate: 60400009 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 84.419005] pc : vop2_setup_dly_for_vp+0x1c/0x270 [ 84.423705] lr : vop2_crtc_atomic_begin+0xe90/0x2bb0 [ 84.428667] sp : ffffffc00c8db850 [ 84.431974] x29: ffffffc00c8db850 x28: 0000000000000001 x27:0000000000000000 [ 84.439101] x26: ffffff8005960080 x25: ffffff8006ae9600 x24:ffffff8005961458 [ 84.446226] x23: ffffffc009842850 x22: ffffff80059600f8 x21:ffffffc009823410 [ 84.453355] x20: ffffff8006ae9500 x19: ffffff8002054800 x18:0000000000000030 [ 84.460485] x17: 393431313a632e32 x16: 706f765f6d72645f x15:ffffffffffffffff [ 84.467614] x14: 0000000000000000 x13: 30393431313a632e x12:32706f765f6d7264 [ 84.474738] x11: 5f706968636b636f x10: ffffffc00a2438d8 x9 :ffffffc008813420 [ 84.481865] x8 : 0000000000000000 x7 : ffffffc00a2438d8 x6 :ffffff8005960080 [ 84.488989] x5 : 0000000000017ff4 x4 : ffffffc009418f68 x3 :0000000000000000 [ 84.496118] x2 : 0000000000000001 x1 : ffffff80072cd280 x0 :ffffff8005961458 [ 84.503246] Call trace: [ 84.505693] vop2_setup_dly_for_vp+0x1c/0x270 [ 84.510051] drm_atomic_helper_commit_planes+0x80/0x210 [ 84.515274] rockchip_drm_atomic_helper_commit_tail_rpm+0x19c/0x2fc [ 84.521540] commit_tail+0xa4/0x180 [ 84.525022] drm_atomic_helper_commit+0x16c/0x190 [ 84.529725] drm_atomic_commit+0xac/0xe0 [ 84.533652] drm_atomic_helper_set_config+0xd8/0x110 [ 84.538609] drm_mode_setcrtc+0x1b4/0x690 [ 84.542617] drm_ioctl_kernel+0xb4/0x100 [ 84.546542] drm_ioctl+0x208/0x440 [ 84.549947] __arm64_sys_ioctl+0xb4/0xdc [ 84.553869] invoke_syscall+0x4c/0x114 [ 84.557617] el0_svc_common.constprop.0+0x54/0x180 [ 84.562407] do_el0_svc+0x20/0x2c [ 84.565724] el0_svc+0x14/0x80 [ 84.568784] el0t_64_sync_handler+0xb0/0xb4 [ 84.572961] el0t_64_sync+0x158/0x15c Change-Id: Ic654f121cc180fb3f57834774eca856c3b87ecc1 Signed-off-by: Andy Yan --- drivers/gpu/drm/rockchip/rockchip_drm_vop2.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index 96fccea8e2ca..35451b2fbf9a 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -12104,9 +12104,9 @@ static void vop2_setup_dly_for_vp(struct vop2_video_port *vp) struct drm_crtc *crtc = &vp->rockchip_crtc.crtc; struct rockchip_crtc_state *vcstate = to_rockchip_crtc_state(crtc->state); struct drm_display_mode *adjusted_mode = &crtc->state->adjusted_mode; - u16 hsync_len = adjusted_mode->crtc_hsync_end - adjusted_mode->crtc_hsync_start; - u16 hdisplay = adjusted_mode->crtc_hdisplay; u32 bg_dly = vp_data->pre_scan_max_dly[0]; + u16 hsync_len; + u16 hdisplay; u32 pre_scan_dly; if (vp_data->hdr_table) { @@ -12131,9 +12131,9 @@ static void vop2_setup_dly_for_vp(struct vop2_video_port *vp) if (vp->splice_mode_right) { vcstate = to_rockchip_crtc_state(left_vp->rockchip_crtc.crtc.state); adjusted_mode = &left_vp->rockchip_crtc.crtc.state->adjusted_mode; - hsync_len = adjusted_mode->crtc_hsync_end - adjusted_mode->crtc_hsync_start; - hdisplay = adjusted_mode->crtc_hdisplay; } + hsync_len = adjusted_mode->crtc_hsync_end - adjusted_mode->crtc_hsync_start; + hdisplay = adjusted_mode->crtc_hdisplay; /* * splice mode: hdisplay must roundup as 4 pixel,