From 1b2222254f78048324c4419b88b8aa4145464bb4 Mon Sep 17 00:00:00 2001 From: Shawn Lin Date: Mon, 19 Aug 2024 08:49:52 +0800 Subject: [PATCH 1/6] phy: rockchip: naneng-combphy: Update external clk parameters for better SI Signed-off-by: Shawn Lin Change-Id: Ic121a29ddc00357b069b27b0fe5e8f4654677ae1 --- .../rockchip/phy-rockchip-naneng-combphy.c | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c index 630c5765d516..2daa3a688ee0 100644 --- a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c +++ b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c @@ -554,9 +554,15 @@ static int rk3528_combphy_cfg(struct rockchip_combphy_priv *priv) if (device_property_read_bool(priv->dev, "rockchip,ext-refclk")) { rockchip_combphy_param_write(priv->phy_grf, &cfg->pipe_clk_ext, true); if (priv->mode == PHY_TYPE_PCIE && rate == 100000000) { - val = readl(priv->mmio + 0x108); - val |= BIT(29) | (0x3 << 4 | 0x1 << 7); - writel(val, priv->mmio + 0x108); + /* + * PLL charge pump current adjust = 111 + * PLL LPF R1 adjust = 1001 + * PLL KVCO adjust = 000 (min) + * PLL KVCO fine tuning signals = 01 + */ + rockchip_combphy_updatel(priv, GENMASK(2, 0), + BIT(29) | (0x7 << 4 | 0x9 << 7), 0x108); + rockchip_combphy_updatel(priv, GENMASK(12, 10), 0x2 << 10, 0x18); } } @@ -890,6 +896,15 @@ static int rk3568_combphy_cfg(struct rockchip_combphy_priv *priv) if (device_property_read_bool(priv->dev, "rockchip,ext-refclk")) { rockchip_combphy_param_write(priv->phy_grf, &cfg->pipe_clk_ext, true); if (priv->mode == PHY_TYPE_PCIE && rate == 100000000) { + /* + * PLL charge pump current adjust = 111 + * PLL LPF R1 adjust = 1001 + * PLL KVCO adjust = 000 (min) + * PLL KVCO fine tuning signals = 01 + */ + rockchip_combphy_updatel(priv, GENMASK(2, 0), 0xf << 4, 0xa << 2); + rockchip_combphy_updatel(priv, GENMASK(2, 0), 0x4, 0xb << 2); + rockchip_combphy_updatel(priv, GENMASK(4, 2), 0x2 << 2, 0x20 << 2); rockchip_combphy_updatel(priv, 0x3 << 4 | 0x1 << 7, 0x3 << 4 | 0x1 << 7, 0xc << 2); From 8de69287b97ced1d727a1ca3922380e6eed5ebf9 Mon Sep 17 00:00:00 2001 From: Algea Cao Date: Mon, 12 Aug 2024 17:03:05 +0800 Subject: [PATCH 2/6] drm/bridge: synopsys: dw-hdmi-qp: Support set ddc scl freq from dts Change-Id: I8a57946cbf70976a87bf19e963581e02a5c0a521 Signed-off-by: Algea Cao --- drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c | 43 ++++++++++++++++---- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c index 4fd6b914d707..b3c19200cd48 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c @@ -1004,9 +1004,23 @@ static const struct hdmi_quirk *get_hdmi_quirk(u8 *vendor_id) static void dw_hdmi_i2c_init(struct dw_hdmi_qp *hdmi) { + u64 scl_high_cnt, scl_low_cnt, val; + + scl_high_cnt = hdmi->i2c->scl_high_ns; + scl_low_cnt = hdmi->i2c->scl_low_ns; + + scl_high_cnt = scl_high_cnt * hdmi->refclk_rate; + scl_high_cnt = DIV_ROUND_CLOSEST_ULL(scl_high_cnt, 1000000000); + + scl_low_cnt = scl_low_cnt * hdmi->refclk_rate; + scl_low_cnt = DIV_ROUND_CLOSEST_ULL(scl_low_cnt, 1000000000); + + val = (scl_high_cnt & 0xffff) << 16 | (scl_low_cnt & 0xffff); + /* Software reset */ hdmi_writel(hdmi, 0x01, I2CM_CONTROL0); + hdmi_writel(hdmi, val, I2CM_SM_SCL_CONFIG0); hdmi_modb(hdmi, 0, I2CM_FM_EN, I2CM_INTERFACE_CONTROL0); /* Clear DONE and ERROR interrupts */ @@ -4311,23 +4325,36 @@ static struct dw_hdmi_qp *dw_hdmi_qp_probe(struct platform_device *pdev, /* If DDC bus is not specified, try to register HDMI I2C bus */ if (!hdmi->ddc) { hdmi->ddc = dw_hdmi_i2c_adapter(hdmi); - if (IS_ERR(hdmi->ddc)) - hdmi->ddc = NULL; + if (IS_ERR(hdmi->ddc)) { + dev_err(hdmi->dev, "register HDMI I2C bus failed\n"); + return ERR_PTR(-ENOMEM); + } + /* * Read high and low time from device tree. If not available use - * the default timing scl clock rate is about 99.6KHz. + * the default timing scl clock rate is about 100KHz. */ if (of_property_read_u32(np, "ddc-i2c-scl-high-time-ns", &hdmi->i2c->scl_high_ns)) - hdmi->i2c->scl_high_ns = 4708; + hdmi->i2c->scl_high_ns = 5000; if (of_property_read_u32(np, "ddc-i2c-scl-low-time-ns", &hdmi->i2c->scl_low_ns)) - hdmi->i2c->scl_low_ns = 4916; - } + hdmi->i2c->scl_low_ns = 5000; - /* Reset HDMI DDC I2C master controller and mute I2CM interrupts */ - if (hdmi->i2c) + /* limit scl freq from 10KHz to 100KHz */ + if (hdmi->i2c->scl_high_ns < 5000 || hdmi->i2c->scl_high_ns > 50000) { + dev_err(hdmi->dev, "scl_high_ns is out of range, set to 5000\n"); + hdmi->i2c->scl_high_ns = 5000; + } + + if (hdmi->i2c->scl_low_ns < 5000 || hdmi->i2c->scl_low_ns > 50000) { + dev_err(hdmi->dev, "scl_low_ns is out of range, set to 5000\n"); + hdmi->i2c->scl_low_ns = 5000; + } + + /* Reset HDMI DDC I2C master controller and mute I2CM interrupts */ dw_hdmi_i2c_init(hdmi); + } init_completion(&hdmi->flt_cmp); init_completion(&hdmi->earc_cmp); From 8d8f23f43553c9716b4cddc02e92b716861c71d6 Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Wed, 10 Jan 2024 10:51:03 +0800 Subject: [PATCH 3/6] MALI: rockchip: upgrade bifrost DDK to g22p0-01eac0, from g21p0-01eac0 Change-Id: I7dabc77e636e7507ee6d3ef7d573e7ea3566703a Signed-off-by: Zhen Chen --- .../dma-buf-test-exporter.c | 18 +- .../memory_group_manager.c | 46 +- .../protected_memory_allocator.c | 2 +- drivers/gpu/arm/bifrost/Kbuild | 2 +- drivers/gpu/arm/bifrost/Kconfig | 12 +- drivers/gpu/arm/bifrost/Makefile | 6 +- .../arm/bifrost/arbiter/mali_kbase_arbif.c | 4 +- .../bifrost/arbiter/mali_kbase_arbiter_pm.c | 2 +- .../gpu/mali_kbase_cache_policy_backend.h | 5 +- .../gpu/mali_kbase_debug_job_fault_backend.c | 46 +- ...mali_kbase_jm_defs.h => mali_kbase_defs.h} | 4 +- .../bifrost/backend/gpu/mali_kbase_devfreq.c | 13 +- .../backend/gpu/mali_kbase_gpuprops_backend.c | 4 +- .../backend/gpu/mali_kbase_instr_backend.c | 27 +- .../backend/gpu/mali_kbase_irq_internal.h | 69 ++- .../backend/gpu/mali_kbase_irq_linux.c | 243 +++++------ .../bifrost/backend/gpu/mali_kbase_jm_hw.c | 62 +-- .../bifrost/backend/gpu/mali_kbase_jm_rb.c | 23 +- .../backend/gpu/mali_kbase_js_backend.c | 12 +- .../backend/gpu/mali_kbase_model_dummy.c | 73 ++-- .../backend/gpu/mali_kbase_model_dummy.h | 6 +- .../gpu/mali_kbase_model_error_generator.c | 2 +- .../backend/gpu/mali_kbase_model_linux.c | 92 ++-- .../backend/gpu/mali_kbase_model_linux.h | 11 +- .../backend/gpu/mali_kbase_pm_backend.c | 104 ++++- .../bifrost/backend/gpu/mali_kbase_pm_ca.c | 12 +- .../bifrost/backend/gpu/mali_kbase_pm_ca.h | 11 + .../bifrost/backend/gpu/mali_kbase_pm_defs.h | 34 +- .../backend/gpu/mali_kbase_pm_driver.c | 65 ++- .../backend/gpu/mali_kbase_pm_internal.h | 1 + .../backend/gpu/mali_kbase_pm_metrics.c | 2 +- .../backend/gpu/mali_kbase_pm_policy.c | 9 +- .../arm/bifrost/backend/gpu/mali_kbase_time.c | 43 +- drivers/gpu/arm/bifrost/build.bp | 3 - .../arm/bifrost/context/mali_kbase_context.c | 2 +- .../arm/bifrost/context/mali_kbase_context.h | 2 +- .../ipa_control/mali_kbase_csf_ipa_control.c | 19 +- drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c | 55 +-- .../bifrost/csf/mali_kbase_csf_csg_debugfs.c | 5 +- .../gpu/arm/bifrost/csf/mali_kbase_csf_defs.h | 102 +++-- .../arm/bifrost/csf/mali_kbase_csf_firmware.c | 230 +++++----- .../arm/bifrost/csf/mali_kbase_csf_firmware.h | 21 +- .../bifrost/csf/mali_kbase_csf_firmware_cfg.c | 10 +- .../bifrost/csf/mali_kbase_csf_firmware_log.c | 13 +- .../csf/mali_kbase_csf_firmware_no_mali.c | 123 +++--- .../gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c | 103 ++--- .../gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h | 13 +- .../csf/mali_kbase_csf_kcpu_fence_debugfs.c | 8 +- .../csf/mali_kbase_csf_mcu_shared_reg.c | 4 +- .../csf/mali_kbase_csf_protected_memory.c | 4 +- .../bifrost/csf/mali_kbase_csf_registers.h | 84 ++-- .../bifrost/csf/mali_kbase_csf_reset_gpu.c | 18 +- .../bifrost/csf/mali_kbase_csf_scheduler.c | 347 +++++++++------ .../bifrost/csf/mali_kbase_csf_scheduler.h | 7 +- .../gpu/arm/bifrost/csf/mali_kbase_csf_sync.c | 4 - .../bifrost/csf/mali_kbase_csf_tiler_heap.c | 4 +- .../csf/mali_kbase_csf_tiler_heap_reclaim.c | 20 +- .../arm/bifrost/csf/mali_kbase_csf_timeout.c | 6 +- .../bifrost/csf/mali_kbase_debug_csf_fault.c | 8 +- .../backend/mali_kbase_debug_coresight_csf.c | 3 +- .../backend/mali_kbase_debug_ktrace_csf.c | 28 +- .../backend/mali_kbase_debug_ktrace_defs_jm.h | 2 +- .../backend/mali_kbase_debug_ktrace_jm.c | 23 +- .../backend/mali_kbase_debug_ktrace_jm.h | 12 +- .../mali_kbase_debug_linux_ktrace_jm.h | 16 +- .../bifrost/debug/mali_kbase_debug_ktrace.c | 14 +- .../device/backend/mali_kbase_device_csf.c | 6 +- .../device/backend/mali_kbase_device_hw_csf.c | 1 + .../device/backend/mali_kbase_device_hw_jm.c | 1 + .../device/backend/mali_kbase_device_jm.c | 6 +- .../arm/bifrost/device/mali_kbase_device.c | 64 ++- .../arm/bifrost/device/mali_kbase_device_hw.c | 8 +- .../gpu/backend/mali_kbase_gpu_fault_csf.c | 16 +- .../bifrost/hw_access/mali_kbase_hw_access.h | 35 ++ .../hw_access/mali_kbase_hw_access_regmap.h | 24 +- .../regmap/mali_kbase_regmap_csf_macros.h | 23 +- .../regmap/mali_kbase_regmap_jm_macros.h | 15 +- .../backend/mali_kbase_hwcnt_backend_csf.c | 57 ++- .../backend/mali_kbase_hwcnt_backend_csf.h | 11 + .../mali_kbase_hwcnt_backend_csf_if_fw.c | 6 +- .../backend/mali_kbase_hwcnt_backend_jm.c | 101 +++-- .../arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.c | 184 +++++--- .../arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h | 69 ++- .../bifrost/hwcnt/mali_kbase_hwcnt_types.c | 34 +- .../bifrost/hwcnt/mali_kbase_hwcnt_types.h | 5 +- .../mali_kbase_ipa_counter_common_csf.c | 6 +- .../mali_kbase_ipa_counter_common_jm.c | 4 +- drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c | 4 +- .../arm/bifrost/ipa/mali_kbase_ipa_debugfs.c | 2 +- .../arm/bifrost/ipa/mali_kbase_ipa_simple.c | 3 + .../gpu/arm/bifrost/jm/mali_kbase_jm_defs.h | 47 +- drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h | 2 +- .../arm/bifrost/mali_base_hwconfig_issues.h | 403 +++++++++++------ drivers/gpu/arm/bifrost/mali_kbase.h | 88 ++-- .../arm/bifrost/mali_kbase_as_fault_debugfs.h | 4 +- .../gpu/arm/bifrost/mali_kbase_cache_policy.c | 3 +- .../gpu/arm/bifrost/mali_kbase_cache_policy.h | 3 +- drivers/gpu/arm/bifrost/mali_kbase_ccswe.c | 5 +- drivers/gpu/arm/bifrost/mali_kbase_ccswe.h | 1 + drivers/gpu/arm/bifrost/mali_kbase_config.h | 4 +- .../arm/bifrost/mali_kbase_config_defaults.h | 31 ++ .../gpu/arm/bifrost/mali_kbase_core_linux.c | 362 +++++++++------ .../gpu/arm/bifrost/mali_kbase_ctx_sched.c | 11 +- .../gpu/arm/bifrost/mali_kbase_ctx_sched.h | 7 +- drivers/gpu/arm/bifrost/mali_kbase_debug.c | 3 +- drivers/gpu/arm/bifrost/mali_kbase_debug.h | 3 + .../arm/bifrost/mali_kbase_debug_job_fault.h | 6 +- .../arm/bifrost/mali_kbase_debug_mem_allocs.h | 4 +- .../arm/bifrost/mali_kbase_debug_mem_view.c | 4 +- .../arm/bifrost/mali_kbase_debug_mem_zones.h | 4 +- .../arm/bifrost/mali_kbase_debugfs_helper.c | 7 +- drivers/gpu/arm/bifrost/mali_kbase_defs.h | 97 ++-- .../arm/bifrost/mali_kbase_disjoint_events.c | 2 +- .../gpu/arm/bifrost/mali_kbase_dummy_job_wa.c | 61 +-- drivers/gpu/arm/bifrost/mali_kbase_fence.c | 12 +- drivers/gpu/arm/bifrost/mali_kbase_fence.h | 30 +- .../gpu/arm/bifrost/mali_kbase_fence_ops.c | 2 +- .../bifrost/mali_kbase_gpu_memory_debugfs.h | 7 +- .../gpu/arm/bifrost/mali_kbase_gpu_metrics.c | 5 + .../gpu/arm/bifrost/mali_kbase_gpu_metrics.h | 7 +- drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c | 10 +- drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h | 5 +- drivers/gpu/arm/bifrost/mali_kbase_gwt.c | 9 +- drivers/gpu/arm/bifrost/mali_kbase_gwt.h | 3 +- drivers/gpu/arm/bifrost/mali_kbase_hw.c | 6 + .../arm/bifrost/mali_kbase_hwaccess_defs.h | 4 +- .../gpu/arm/bifrost/mali_kbase_hwaccess_jm.h | 2 +- .../gpu/arm/bifrost/mali_kbase_hwaccess_pm.h | 9 +- .../arm/bifrost/mali_kbase_hwaccess_time.h | 9 + drivers/gpu/arm/bifrost/mali_kbase_jd.c | 12 +- drivers/gpu/arm/bifrost/mali_kbase_jm.c | 6 +- drivers/gpu/arm/bifrost/mali_kbase_jm.h | 2 +- drivers/gpu/arm/bifrost/mali_kbase_js.c | 45 +- .../gpu/arm/bifrost/mali_kbase_kinstr_jm.c | 12 +- .../arm/bifrost/mali_kbase_kinstr_prfcnt.c | 109 +++-- drivers/gpu/arm/bifrost/mali_kbase_mem.c | 202 +++++---- drivers/gpu/arm/bifrost/mali_kbase_mem.h | 122 +++--- .../gpu/arm/bifrost/mali_kbase_mem_linux.c | 97 ++-- .../gpu/arm/bifrost/mali_kbase_mem_lowlevel.h | 34 +- .../gpu/arm/bifrost/mali_kbase_mem_migrate.c | 4 +- .../gpu/arm/bifrost/mali_kbase_mem_migrate.h | 8 + drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c | 39 +- .../arm/bifrost/mali_kbase_mem_pool_group.c | 4 +- .../arm/bifrost/mali_kbase_mem_pool_group.h | 4 +- .../bifrost/mali_kbase_mem_profile_debugfs.h | 2 + .../gpu/arm/bifrost/mali_kbase_native_mgm.c | 22 +- drivers/gpu/arm/bifrost/mali_kbase_pbha.c | 9 +- drivers/gpu/arm/bifrost/mali_kbase_pbha.h | 4 +- .../gpu/arm/bifrost/mali_kbase_pbha_debugfs.c | 4 +- .../gpu/arm/bifrost/mali_kbase_pbha_debugfs.h | 4 +- drivers/gpu/arm/bifrost/mali_kbase_pm.c | 8 +- drivers/gpu/arm/bifrost/mali_kbase_pm.h | 4 +- .../arm/bifrost/mali_kbase_refcount_defs.h | 57 --- .../gpu/arm/bifrost/mali_kbase_reg_track.c | 6 +- .../bifrost/mali_kbase_regs_history_debugfs.c | 4 +- .../gpu/arm/bifrost/mali_kbase_reset_gpu.h | 4 +- drivers/gpu/arm/bifrost/mali_kbase_smc.h | 8 +- drivers/gpu/arm/bifrost/mali_kbase_softjobs.c | 16 +- drivers/gpu/arm/bifrost/mali_kbase_sync.h | 11 +- .../gpu/arm/bifrost/mali_kbase_sync_file.c | 65 +-- .../bifrost/mmu/backend/mali_kbase_mmu_csf.c | 95 ++-- .../bifrost/mmu/backend/mali_kbase_mmu_jm.c | 63 ++- drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c | 413 +++++++++--------- drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h | 11 +- .../gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h | 25 +- .../bifrost/mmu/mali_kbase_mmu_hw_direct.c | 133 +++--- .../bifrost/mmu/mali_kbase_mmu_mode_aarch64.c | 4 +- .../platform/meson/mali_kbase_runtime_pm.c | 2 +- .../include/kutf/kutf_kprobe.h} | 18 +- drivers/gpu/arm/bifrost/tests/kutf/Kbuild | 5 +- drivers/gpu/arm/bifrost/tests/kutf/build.bp | 3 +- .../bifrost/tests/kutf/kutf_helpers_user.c | 32 +- .../gpu/arm/bifrost/tests/kutf/kutf_kprobe.c | 344 +++++++++++++++ .../gpu/arm/bifrost/tests/kutf/kutf_suite.c | 25 +- .../gpu/arm/bifrost/tests/kutf/kutf_utils.c | 4 +- .../kernel/mali_kutf_clk_rate_trace_test.c | 8 +- .../mali_kutf_irq_test_main.c | 54 +-- .../mali_kutf_mgm_integration_test_main.c | 2 +- .../arm/bifrost/thirdparty/mali_kbase_mmap.c | 4 +- .../tl/backend/mali_kbase_timeline_csf.c | 6 +- .../tl/backend/mali_kbase_timeline_jm.c | 2 +- .../gpu/arm/bifrost/tl/mali_kbase_timeline.c | 6 +- .../arm/bifrost/tl/mali_kbase_timeline_io.c | 16 +- .../gpu/arm/bifrost/tl/mali_kbase_tlstream.c | 8 +- .../arm/bifrost/tl/mali_kbase_tracepoints.h | 268 ++++++------ include/linux/memory_group_manager.h | 29 +- include/linux/priority_control_manager.h | 82 +++- include/linux/version_compat_defs.h | 51 +++ .../dma-buf-test-exporter.h | 2 +- .../backend/gpu/mali_kbase_model_dummy.h | 2 +- .../backend/gpu/mali_kbase_model_linux.h | 3 +- .../arm/bifrost/csf/mali_base_csf_kernel.h | 10 +- .../arm/bifrost/csf/mali_kbase_csf_ioctl.h | 7 +- .../bifrost/gpu/mali_kbase_gpu_coherency.h | 8 +- .../gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h | 7 +- .../gpu/arm/bifrost/mali_base_common_kernel.h | 4 +- .../uapi/gpu/arm/bifrost/mali_base_kernel.h | 18 +- .../mali_kbase_mem_profile_debugfs_buf_size.h | 4 +- 198 files changed, 4164 insertions(+), 2887 deletions(-) rename drivers/gpu/arm/bifrost/backend/gpu/{mali_kbase_jm_defs.h => mali_kbase_defs.h} (97%) delete mode 100644 drivers/gpu/arm/bifrost/mali_kbase_refcount_defs.h rename drivers/gpu/arm/bifrost/{backend/gpu/mali_kbase_backend_config.h => tests/include/kutf/kutf_kprobe.h} (67%) create mode 100644 drivers/gpu/arm/bifrost/tests/kutf/kutf_kprobe.c diff --git a/drivers/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.c b/drivers/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.c index deef790dc73b..9bf7f8d2dd6d 100644 --- a/drivers/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.c +++ b/drivers/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.c @@ -182,7 +182,7 @@ static struct sg_table *dma_buf_te_map(struct dma_buf_attachment *attachment, sg_set_page(iter, alloc->pages[i], PAGE_SIZE, 0); } - if (!dma_map_sg(attachment->dev, sg->sgl, sg->nents, direction)) { + if (!dma_map_sg(attachment->dev, sg->sgl, (int)sg->nents, direction)) { mutex_unlock(&alloc->lock); sg_free_table(sg); kfree(sg); @@ -213,7 +213,7 @@ static void dma_buf_te_unmap(struct dma_buf_attachment *attachment, struct sg_ta pa->sg = NULL; mutex_unlock(&alloc->lock); - dma_unmap_sg(attachment->dev, sg->sgl, sg->nents, direction); + dma_unmap_sg(attachment->dev, sg->sgl, (int)sg->nents, direction); sg_free_table(sg); kfree(sg); } @@ -273,12 +273,12 @@ static int dma_buf_te_sync(struct dma_buf *dmabuf, enum dma_data_direction direc dev_dbg(te_device.this_device, "sync cpu with device %s\n", dev_name(attachment->dev)); - dma_sync_sg_for_cpu(attachment->dev, sg->sgl, sg->nents, direction); + dma_sync_sg_for_cpu(attachment->dev, sg->sgl, (int)sg->nents, direction); } else { dev_dbg(te_device.this_device, "sync device %s with cpu\n", dev_name(attachment->dev)); - dma_sync_sg_for_device(attachment->dev, sg->sgl, sg->nents, direction); + dma_sync_sg_for_device(attachment->dev, sg->sgl, (int)sg->nents, direction); } } @@ -682,7 +682,7 @@ err_have_dmabuf: return res; } -static u32 dma_te_buf_fill(struct dma_buf *dma_buf, unsigned int value) +static int dma_te_buf_fill(struct dma_buf *dma_buf, int value) { struct dma_buf_attachment *attachment; struct sg_table *sgt; @@ -695,7 +695,11 @@ static u32 dma_te_buf_fill(struct dma_buf *dma_buf, unsigned int value) if (IS_ERR_OR_NULL(attachment)) return -EBUSY; +#if (KERNEL_VERSION(6, 1, 55) <= LINUX_VERSION_CODE) + sgt = dma_buf_map_attachment_unlocked(attachment, DMA_BIDIRECTIONAL); +#else sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL); +#endif if (IS_ERR_OR_NULL(sgt)) { ret = PTR_ERR(sgt); goto no_import; @@ -729,7 +733,11 @@ static u32 dma_te_buf_fill(struct dma_buf *dma_buf, unsigned int value) no_kmap: dma_buf_end_cpu_access(dma_buf, DMA_BIDIRECTIONAL); no_cpu_access: +#if (KERNEL_VERSION(6, 1, 55) <= LINUX_VERSION_CODE) + dma_buf_unmap_attachment_unlocked(attachment, sgt, DMA_BIDIRECTIONAL); +#else dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL); +#endif no_import: dma_buf_detach(dma_buf, attachment); return ret; diff --git a/drivers/base/arm/memory_group_manager/memory_group_manager.c b/drivers/base/arm/memory_group_manager/memory_group_manager.c index 11dc1c2e24d8..389b0f051f3a 100644 --- a/drivers/base/arm/memory_group_manager/memory_group_manager.c +++ b/drivers/base/arm/memory_group_manager/memory_group_manager.c @@ -61,7 +61,7 @@ static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigne * struct mgm_group - Structure to keep track of the number of allocated * pages per group * - * @size: The number of allocated small(4KB) pages + * @size: The number of allocated small pages of PAGE_SIZE bytes * @lp_size: The number of allocated large(2MB) pages * @insert_pfn: The number of calls to map pages for CPU access. * @update_gpu_pte: The number of calls to update GPU page table entries. @@ -100,7 +100,7 @@ static int mgm_size_get(void *data, u64 *val) { struct mgm_group *group = data; - *val = atomic_read(&group->size); + *val = (u64)atomic_read(&group->size); return 0; } @@ -108,21 +108,21 @@ static int mgm_size_get(void *data, u64 *val) static int mgm_lp_size_get(void *data, u64 *val) { struct mgm_group *group = data; - *val = atomic_read(&group->lp_size); + *val = (u64)atomic_read(&group->lp_size); return 0; } static int mgm_insert_pfn_get(void *data, u64 *val) { struct mgm_group *group = data; - *val = atomic_read(&group->insert_pfn); + *val = (u64)atomic_read(&group->insert_pfn); return 0; } static int mgm_update_gpu_pte_get(void *data, u64 *val) { struct mgm_group *group = data; - *val = atomic_read(&group->update_gpu_pte); + *val = (u64)atomic_read(&group->update_gpu_pte); return 0; } @@ -212,9 +212,9 @@ static int mgm_initialize_debugfs(struct mgm_groups *mgm_data) #endif /* CONFIG_DEBUG_FS */ #define ORDER_SMALL_PAGE 0 -#define ORDER_LARGE_PAGE 9 +#define ORDER_LARGE_PAGE (__builtin_ffs(SZ_2M / PAGE_SIZE) - 1) static void update_size(struct memory_group_manager_device *mgm_dev, unsigned int group_id, - int order, bool alloc) + unsigned int order, bool alloc) { struct mgm_groups *data = mgm_dev->data; @@ -238,21 +238,22 @@ static void update_size(struct memory_group_manager_device *mgm_dev, unsigned in break; default: - dev_err(data->dev, "Unknown order(%d)\n", order); + dev_err(data->dev, "Unknown order(%u)\n", order); break; } } static struct page *example_mgm_alloc_page(struct memory_group_manager_device *mgm_dev, - int group_id, gfp_t gfp_mask, unsigned int order) + unsigned int group_id, gfp_t gfp_mask, + unsigned int order) { struct mgm_groups *const data = mgm_dev->data; struct page *p; - dev_dbg(data->dev, "%s(mgm_dev=%pK, group_id=%d gfp_mask=0x%x order=%u\n", __func__, + dev_dbg(data->dev, "%s(mgm_dev=%pK, group_id=%u gfp_mask=0x%x order=%u\n", __func__, (void *)mgm_dev, group_id, gfp_mask, order); - if (WARN_ON(group_id < 0) || WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return NULL; p = alloc_pages(gfp_mask, order); @@ -268,15 +269,15 @@ static struct page *example_mgm_alloc_page(struct memory_group_manager_device *m return p; } -static void example_mgm_free_page(struct memory_group_manager_device *mgm_dev, int group_id, - struct page *page, unsigned int order) +static void example_mgm_free_page(struct memory_group_manager_device *mgm_dev, + unsigned int group_id, struct page *page, unsigned int order) { struct mgm_groups *const data = mgm_dev->data; - dev_dbg(data->dev, "%s(mgm_dev=%pK, group_id=%d page=%pK order=%u\n", __func__, + dev_dbg(data->dev, "%s(mgm_dev=%pK, group_id=%u page=%pK order=%u\n", __func__, (void *)mgm_dev, group_id, (void *)page, order); - if (WARN_ON(group_id < 0) || WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return; __free_pages(page, order); @@ -302,14 +303,14 @@ static int example_mgm_get_import_memory_id(struct memory_group_manager_device * } static u64 example_mgm_update_gpu_pte(struct memory_group_manager_device *const mgm_dev, - int const group_id, int const mmu_level, u64 pte) + unsigned int const group_id, int const mmu_level, u64 pte) { struct mgm_groups *const data = mgm_dev->data; - dev_dbg(data->dev, "%s(mgm_dev=%pK, group_id=%d, mmu_level=%d, pte=0x%llx)\n", __func__, + dev_dbg(data->dev, "%s(mgm_dev=%pK, group_id=%u, mmu_level=%d, pte=0x%llx)\n", __func__, (void *)mgm_dev, group_id, mmu_level, pte); - if (WARN_ON(group_id < 0) || WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return pte; pte |= ((u64)group_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; @@ -323,7 +324,8 @@ static u64 example_mgm_update_gpu_pte(struct memory_group_manager_device *const } static u64 example_mgm_pte_to_original_pte(struct memory_group_manager_device *const mgm_dev, - int const group_id, int const mmu_level, u64 pte) + unsigned int const group_id, int const mmu_level, + u64 pte) { CSTD_UNUSED(mgm_dev); CSTD_UNUSED(group_id); @@ -338,7 +340,7 @@ static u64 example_mgm_pte_to_original_pte(struct memory_group_manager_device *c } static vm_fault_t example_mgm_vmf_insert_pfn_prot(struct memory_group_manager_device *const mgm_dev, - int const group_id, + unsigned int const group_id, struct vm_area_struct *const vma, unsigned long const addr, unsigned long const pfn, pgprot_t const prot) @@ -347,11 +349,11 @@ static vm_fault_t example_mgm_vmf_insert_pfn_prot(struct memory_group_manager_de vm_fault_t fault; dev_dbg(data->dev, - "%s(mgm_dev=%pK, group_id=%d, vma=%pK, addr=0x%lx, pfn=0x%lx, prot=0x%llx)\n", + "%s(mgm_dev=%pK, group_id=%u, vma=%pK, addr=0x%lx, pfn=0x%lx, prot=0x%llx)\n", __func__, (void *)mgm_dev, group_id, (void *)vma, addr, pfn, (unsigned long long)pgprot_val(prot)); - if (WARN_ON(group_id < 0) || WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) + if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return VM_FAULT_SIGBUS; fault = vmf_insert_pfn_prot(vma, addr, pfn, prot); diff --git a/drivers/base/arm/protected_memory_allocator/protected_memory_allocator.c b/drivers/base/arm/protected_memory_allocator/protected_memory_allocator.c index d7e0bec0978e..b27eed93fc8b 100644 --- a/drivers/base/arm/protected_memory_allocator/protected_memory_allocator.c +++ b/drivers/base/arm/protected_memory_allocator/protected_memory_allocator.c @@ -303,7 +303,7 @@ simple_pma_alloc_page(struct protected_memory_allocator_device *pma_dev, unsigne large_granularity_alloc(epma_dev, start_idx, order, pma); - epma_dev->num_free_pages -= 1 << order; + epma_dev->num_free_pages -= 1ULL << order; spin_unlock(&epma_dev->rmem_lock); return pma; } diff --git a/drivers/gpu/arm/bifrost/Kbuild b/drivers/gpu/arm/bifrost/Kbuild index 957f412d6547..b35fcee88baa 100644 --- a/drivers/gpu/arm/bifrost/Kbuild +++ b/drivers/gpu/arm/bifrost/Kbuild @@ -69,7 +69,7 @@ endif # # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= '"g21p0-01eac0"' +MALI_RELEASE_NAME ?= '"g22p0-01eac0"' # Set up defaults if not defined by build system ifeq ($(CONFIG_MALI_BIFROST_DEBUG), y) MALI_UNIT_TEST = 1 diff --git a/drivers/gpu/arm/bifrost/Kconfig b/drivers/gpu/arm/bifrost/Kconfig index f32b107949ba..6880b39323d6 100644 --- a/drivers/gpu/arm/bifrost/Kconfig +++ b/drivers/gpu/arm/bifrost/Kconfig @@ -121,7 +121,7 @@ config MALI_BIFROST_ENABLE_TRACE config MALI_ARBITER_SUPPORT bool "Enable arbiter support for Mali" - depends on MALI_BIFROST && !MALI_CSF_SUPPORT + depends on MALI_BIFROST default n help Enable support for the arbiter interface in the driver. @@ -201,16 +201,6 @@ config PAGE_MIGRATION_SUPPORT If in doubt, say Y. To strip out page migration symbols and support, say N. -config MALI_MEMORY_FULLY_BACKED - bool "Enable memory fully physically-backed" - depends on MALI_BIFROST && MALI_BIFROST_EXPERT - default n - help - This option enables full physical backing of all virtual - memory allocations in the kernel. Notice that this build - option only affects allocations of grow-on-GPU-page-fault - memory. - config MALI_CORESTACK bool "Enable support of GPU core stack power control" depends on MALI_BIFROST && MALI_BIFROST_EXPERT diff --git a/drivers/gpu/arm/bifrost/Makefile b/drivers/gpu/arm/bifrost/Makefile index f0f5e6072193..69dbe3750a10 100644 --- a/drivers/gpu/arm/bifrost/Makefile +++ b/drivers/gpu/arm/bifrost/Makefile @@ -105,7 +105,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_CORESTACK = n CONFIG_LARGE_PAGE_SUPPORT = y CONFIG_MALI_PWRSOFT_765 = n - CONFIG_MALI_MEMORY_FULLY_BACKED = n CONFIG_MALI_JOB_DUMP = n CONFIG_MALI_BIFROST_NO_MALI = n CONFIG_MALI_REAL_HW = y @@ -170,7 +169,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_CORESTACK \ CONFIG_LARGE_PAGE_SUPPORT \ CONFIG_MALI_PWRSOFT_765 \ - CONFIG_MALI_MEMORY_FULLY_BACKED \ CONFIG_MALI_JOB_DUMP \ CONFIG_MALI_BIFROST_NO_MALI \ CONFIG_MALI_BIFROST_ERROR_INJECT \ @@ -193,6 +191,7 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD + endif THIS_DIR := $(dir $(lastword $(MAKEFILE_LIST))) @@ -231,7 +230,8 @@ endif # # KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions # -BASE_SYMBOLS = $(M)/../../base/arm/Module.symvers +BASE_SYMBOLS = + EXTRA_SYMBOLS += \ $(BASE_SYMBOLS) diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c index 728f013f293d..c290dd6b086f 100644 --- a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c @@ -183,7 +183,9 @@ int kbase_arbif_init(struct kbase_device *kbdev) dev_dbg(kbdev->dev, "%s\n", __func__); - arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter_if", 0); + arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter-if", 0); + if (!arbiter_if_node) + arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter_if", 0); if (!arbiter_if_node) { dev_dbg(kbdev->dev, "No arbiter_if in Device Tree\n"); /* no arbiter interface defined in device tree */ diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c index 4498b469300e..616b0a78cbe5 100644 --- a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c @@ -308,7 +308,7 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) err = wait_event_timeout(arb_vm_state->vm_state_wait, arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU, - msecs_to_jiffies(gpu_req_timeout)); + msecs_to_jiffies((unsigned int)gpu_req_timeout)); if (!err) { dev_dbg(kbdev->dev, diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h index d95aa37d8950..7317f1449d44 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_cache_policy_backend.h @@ -22,8 +22,9 @@ #ifndef _KBASE_CACHE_POLICY_BACKEND_H_ #define _KBASE_CACHE_POLICY_BACKEND_H_ -#include "mali_kbase.h" -#include +#include + +struct kbase_device; /** * kbase_cache_set_coherency_mode() - Sets the system coherency mode diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c index bd3622e7d44b..af8d1e3af87c 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c @@ -27,34 +27,38 @@ #if IS_ENABLED(CONFIG_DEBUG_FS) /*GPU_CONTROL_REG(r)*/ -static int gpu_control_reg_snapshot[] = { GPU_CONTROL_ENUM(GPU_ID), GPU_CONTROL_ENUM(SHADER_READY), - GPU_CONTROL_ENUM(TILER_READY), - GPU_CONTROL_ENUM(L2_READY) }; +static unsigned int gpu_control_reg_snapshot[] = { GPU_CONTROL_ENUM(GPU_ID), + GPU_CONTROL_ENUM(SHADER_READY), + GPU_CONTROL_ENUM(TILER_READY), + GPU_CONTROL_ENUM(L2_READY) }; /* JOB_CONTROL_REG(r) */ -static int job_control_reg_snapshot[] = { JOB_CONTROL_ENUM(JOB_IRQ_MASK), - JOB_CONTROL_ENUM(JOB_IRQ_STATUS) }; +static unsigned int job_control_reg_snapshot[] = { JOB_CONTROL_ENUM(JOB_IRQ_MASK), + JOB_CONTROL_ENUM(JOB_IRQ_STATUS) }; /* JOB_SLOT_REG(n,r) */ -static int job_slot_reg_snapshot[] = { JOB_SLOT_ENUM(0, HEAD) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, TAIL) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, AFFINITY) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, CONFIG) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, STATUS) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, HEAD_NEXT) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, AFFINITY_NEXT) - JOB_SLOT_BASE_ENUM(0), - JOB_SLOT_ENUM(0, CONFIG_NEXT) - JOB_SLOT_BASE_ENUM(0) }; +static unsigned int job_slot_reg_snapshot[] = { + JOB_SLOT_ENUM(0, HEAD) - JOB_SLOT_BASE_ENUM(0), + JOB_SLOT_ENUM(0, TAIL) - JOB_SLOT_BASE_ENUM(0), + JOB_SLOT_ENUM(0, AFFINITY) - JOB_SLOT_BASE_ENUM(0), + JOB_SLOT_ENUM(0, CONFIG) - JOB_SLOT_BASE_ENUM(0), + JOB_SLOT_ENUM(0, STATUS) - JOB_SLOT_BASE_ENUM(0), + JOB_SLOT_ENUM(0, HEAD_NEXT) - JOB_SLOT_BASE_ENUM(0), + JOB_SLOT_ENUM(0, AFFINITY_NEXT) - JOB_SLOT_BASE_ENUM(0), + JOB_SLOT_ENUM(0, CONFIG_NEXT) - JOB_SLOT_BASE_ENUM(0) +}; /*MMU_CONTROL_REG(r)*/ -static int mmu_reg_snapshot[] = { MMU_CONTROL_ENUM(IRQ_MASK), MMU_CONTROL_ENUM(IRQ_STATUS) }; +static unsigned int mmu_reg_snapshot[] = { MMU_CONTROL_ENUM(IRQ_MASK), + MMU_CONTROL_ENUM(IRQ_STATUS) }; /* MMU_AS_REG(n,r) */ -static int as_reg_snapshot[] = { MMU_AS_ENUM(0, TRANSTAB) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, TRANSCFG) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, MEMATTR) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, FAULTSTATUS) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, FAULTADDRESS) - MMU_AS_BASE_ENUM(0), - MMU_AS_ENUM(0, STATUS) - MMU_AS_BASE_ENUM(0) }; +static unsigned int as_reg_snapshot[] = { MMU_AS_ENUM(0, TRANSTAB) - MMU_AS_BASE_ENUM(0), + MMU_AS_ENUM(0, TRANSCFG) - MMU_AS_BASE_ENUM(0), + MMU_AS_ENUM(0, MEMATTR) - MMU_AS_BASE_ENUM(0), + MMU_AS_ENUM(0, FAULTSTATUS) - MMU_AS_BASE_ENUM(0), + MMU_AS_ENUM(0, FAULTADDRESS) - MMU_AS_BASE_ENUM(0), + MMU_AS_ENUM(0, STATUS) - MMU_AS_BASE_ENUM(0) }; bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, int reg_range) { @@ -132,7 +136,7 @@ bool kbase_debug_job_fault_reg_snapshot_init(struct kbase_context *kctx, int reg bool kbase_job_fault_get_reg_snapshot(struct kbase_context *kctx) { int offset = 0; - int reg_enum; + u32 reg_enum; u64 val64; if (kctx->reg_dump == NULL) diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_defs.h similarity index 97% rename from drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h rename to drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_defs.h index e9dbe825d47f..bff0b7235736 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_defs.h @@ -50,7 +50,7 @@ struct rb_entry { * u64 for serving as tagged value. * @kctx: Pointer to kbase context. */ -#define SLOT_RB_TAG_KCTX(kctx) (u64)((uintptr_t)(kctx)) +#define SLOT_RB_TAG_KCTX(kctx) ((u64)(uintptr_t)(kctx)) /** * struct slot_rb - Slot ringbuffer * @entries: Ringbuffer entries @@ -116,7 +116,7 @@ struct kbase_backend_data { * within the timeout period */ #define KBASE_RESET_GPU_COMMITTED 2 -/* The GPU reset process is currently occuring (timeout has expired or +/* The GPU reset process is currently occurring (timeout has expired or * kbasep_try_reset_gpu_early was called) */ #define KBASE_RESET_GPU_HAPPENING 3 diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c index 0712439e37a5..2649f1815e9f 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c @@ -197,7 +197,7 @@ static int kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *s static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, struct devfreq_dev_profile *dp) { int count; - int i = 0; + unsigned int i = 0; unsigned long freq; struct dev_pm_opp *opp; @@ -211,14 +211,14 @@ static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, struct devf if (count < 0) return count; - dp->freq_table = kmalloc_array(count, sizeof(dp->freq_table[0]), GFP_KERNEL); + dp->freq_table = kmalloc_array((size_t)count, sizeof(dp->freq_table[0]), GFP_KERNEL); if (!dp->freq_table) return -ENOMEM; #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE rcu_read_lock(); #endif - for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) { + for (i = 0, freq = ULONG_MAX; i < (unsigned int)count; i++, freq--) { opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq); if (IS_ERR(opp)) break; @@ -232,8 +232,8 @@ static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, struct devf rcu_read_unlock(); #endif - if (count != i) - dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%d\n", count, i); + if ((unsigned int)count != i) + dev_warn(kbdev->dev, "Unable to enumerate all OPPs (%d!=%u\n", count, i); dp->max_state = i; @@ -335,7 +335,8 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) return 0; count = dev_pm_opp_get_opp_count(kbdev->dev); - kbdev->devfreq_table = kmalloc_array(count, sizeof(struct kbase_devfreq_opp), GFP_KERNEL); + kbdev->devfreq_table = + kmalloc_array((size_t)count, sizeof(struct kbase_devfreq_opp), GFP_KERNEL); if (!kbdev->devfreq_table) return -ENOMEM; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c index 443a1466e0e0..414ad546811a 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c @@ -30,7 +30,7 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprops_regdump *regdump) { - int i; + uint i; /* regdump is zero intiialized, individual entries do not need to be explicitly set */ regdump->gpu_id = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_ID)); @@ -122,7 +122,7 @@ int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, #if MALI_USE_CSF if (kbase_hw_has_l2_slice_hash_feature(kbdev)) { - int i; + uint i; for (i = 0; i < GPU_L2_SLICE_HASH_COUNT; i++) regdump->l2_slice_hash[i] = kbase_reg_read32(kbdev, GPU_L2_SLICE_HASH_OFFSET(i)); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c index 3d61081e0f84..131cfe32df9f 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c @@ -31,17 +31,17 @@ static int wait_prfcnt_ready(struct kbase_device *kbdev) { - u32 loops; - - for (loops = 0; loops < KBASE_PRFCNT_ACTIVE_MAX_LOOPS; loops++) { - const u32 prfcnt_active = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) & - GPU_STATUS_PRFCNT_ACTIVE; - if (!prfcnt_active) - return 0; + u32 val; + const u32 timeout_us = + kbase_get_timeout_ms(kbdev, KBASE_PRFCNT_ACTIVE_TIMEOUT) * USEC_PER_MSEC; + const int err = kbase_reg_poll32_timeout(kbdev, GPU_CONTROL_ENUM(GPU_STATUS), val, + !(val & GPU_STATUS_PRFCNT_ACTIVE), 0, timeout_us, + false); + if (err) { + dev_err(kbdev->dev, "PRFCNT_ACTIVE bit stuck\n"); + return -EBUSY; } - - dev_err(kbdev->dev, "PRFCNT_ACTIVE bit stuck\n"); - return -EBUSY; + return 0; } int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, struct kbase_context *kctx, @@ -86,11 +86,12 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, struct kbase_c spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); /* Configure */ - prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; + prfcnt_config = (u32)kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; #ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS - prfcnt_config |= kbdev->hwcnt.backend.override_counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT; + prfcnt_config |= (u32)kbdev->hwcnt.backend.override_counter_set + << PRFCNT_CONFIG_SETSELECT_SHIFT; #else - prfcnt_config |= enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT; + prfcnt_config |= (u32)enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT; #endif /* Wait until prfcnt config register can be written */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h index 1429c01a1bd2..34e8178d1d76 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_internal.h @@ -26,8 +26,29 @@ #ifndef _KBASE_IRQ_INTERNAL_H_ #define _KBASE_IRQ_INTERNAL_H_ +/* GPU IRQ Tags */ +#define JOB_IRQ_TAG 0 +#define MMU_IRQ_TAG 1 +#define GPU_IRQ_TAG 2 + +/** + * kbase_install_interrupts - Install IRQs handlers. + * + * @kbdev: The kbase device + * + * This function must be called once only when a kbase device is initialized. + * + * Return: 0 on success. Error code (negative) on failure. + */ int kbase_install_interrupts(struct kbase_device *kbdev); +/** + * kbase_release_interrupts - Uninstall IRQs handlers. + * + * @kbdev: The kbase device + * + * This function needs to be called when a kbase device is terminated. + */ void kbase_release_interrupts(struct kbase_device *kbdev); /** @@ -37,10 +58,52 @@ void kbase_release_interrupts(struct kbase_device *kbdev); */ void kbase_synchronize_irqs(struct kbase_device *kbdev); -int kbasep_common_test_interrupt_handlers(struct kbase_device *const kbdev); +#ifdef CONFIG_MALI_BIFROST_DEBUG +#if IS_ENABLED(CONFIG_MALI_REAL_HW) +/** + * kbase_validate_interrupts - Validate interrupts + * + * @kbdev: The kbase device + * + * This function will be called once when a kbase device is initialized + * to check whether interrupt handlers are configured appropriately. + * If interrupt numbers and/or flags defined in the device tree are + * incorrect, then the validation might fail. + * The whold device initialization will fail if it returns error code. + * + * Return: 0 on success. Error code (negative) on failure. + */ +int kbase_validate_interrupts(struct kbase_device *const kbdev); +#endif /* CONFIG_MALI_REAL_HW */ +#endif /* CONFIG_MALI_BIFROST_DEBUG */ -irqreturn_t kbase_gpu_irq_test_handler(int irq, void *data, u32 val); +/** + * kbase_get_interrupt_handler - Return default interrupt handler + * @kbdev: Kbase device + * @irq_tag: Tag to choose the handler + * + * If single interrupt line is used the combined interrupt handler + * will be returned regardless of irq_tag. Otherwise the corresponding + * interrupt handler will be returned. + * + * Return: Interrupt handler corresponding to the tag. NULL on failure. + */ +irq_handler_t kbase_get_interrupt_handler(struct kbase_device *kbdev, u32 irq_tag); + +/** + * kbase_set_custom_irq_handler - Set a custom IRQ handler + * + * @kbdev: The kbase device for which the handler is to be registered + * @custom_handler: Handler to be registered + * @irq_tag: Interrupt tag + * + * Register given interrupt handler for requested interrupt tag + * In the case where irq handler is not specified, the default handler shall be + * registered + * + * Return: 0 case success, error code otherwise + */ int kbase_set_custom_irq_handler(struct kbase_device *kbdev, irq_handler_t custom_handler, - int irq_type); + u32 irq_tag); #endif /* _KBASE_IRQ_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c index 6474f27dafcc..9cb367508dde 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c @@ -26,20 +26,15 @@ #include #if IS_ENABLED(CONFIG_MALI_REAL_HW) - -/* GPU IRQ Tags */ -#define JOB_IRQ_TAG 0 -#define MMU_IRQ_TAG 1 -#define GPU_IRQ_TAG 2 - static void *kbase_tag(void *ptr, u32 tag) { return (void *)(((uintptr_t)ptr) | tag); } +#endif static void *kbase_untag(void *ptr) { - return (void *)(((uintptr_t)ptr) & ~3); + return (void *)(((uintptr_t)ptr) & ~(uintptr_t)3); } static irqreturn_t kbase_job_irq_handler(int irq, void *data) @@ -58,12 +53,6 @@ static irqreturn_t kbase_job_irq_handler(int irq, void *data) val = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_STATUS)); -#ifdef CONFIG_MALI_BIFROST_DEBUG - if (!kbdev->pm.backend.driver_ready_for_irqs) - dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", __func__, - irq, val); -#endif /* CONFIG_MALI_BIFROST_DEBUG */ - if (!val) { spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return IRQ_NONE; @@ -101,11 +90,6 @@ static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) val = kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_STATUS)); -#ifdef CONFIG_MALI_BIFROST_DEBUG - if (!kbdev->pm.backend.driver_ready_for_irqs) - dev_warn(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", __func__, - irq, val); -#endif /* CONFIG_MALI_BIFROST_DEBUG */ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); if (!val) { @@ -122,7 +106,8 @@ static irqreturn_t kbase_mmu_irq_handler(int irq, void *data) return IRQ_HANDLED; } -static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) + +static irqreturn_t kbase_gpuonly_irq_handler(int irq, void *data) { unsigned long flags; struct kbase_device *kbdev = kbase_untag(data); @@ -139,13 +124,6 @@ static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) gpu_irq_status = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_STATUS)); -#ifdef CONFIG_MALI_BIFROST_DEBUG - if (!kbdev->pm.backend.driver_ready_for_irqs) { - dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x before driver is ready\n", __func__, - irq, gpu_irq_status); - } -#endif /* CONFIG_MALI_BIFROST_DEBUG */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); if (gpu_irq_status) { @@ -158,82 +136,91 @@ static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) return irq_state; } +/** + * kbase_gpu_irq_handler - GPU interrupt handler + * @irq: IRQ number + * @data: Data associated with this IRQ (i.e. kbdev) + * + * Return: IRQ_HANDLED if any interrupt request has been successfully handled. + * IRQ_NONE otherwise. + */ +static irqreturn_t kbase_gpu_irq_handler(int irq, void *data) +{ + irqreturn_t irq_state = kbase_gpuonly_irq_handler(irq, data); + return irq_state; +} + +/** + * kbase_combined_irq_handler - Combined interrupt handler for all interrupts + * @irq: IRQ number + * @data: Data associated with this IRQ (i.e. kbdev) + * + * This handler will be used for the GPU with single interrupt line. + * + * Return: IRQ_HANDLED if any interrupt request has been successfully handled. + * IRQ_NONE otherwise. + */ +static irqreturn_t kbase_combined_irq_handler(int irq, void *data) +{ + irqreturn_t irq_state = IRQ_NONE; + + if (kbase_job_irq_handler(irq, data) == IRQ_HANDLED) + irq_state = IRQ_HANDLED; + if (kbase_mmu_irq_handler(irq, data) == IRQ_HANDLED) + irq_state = IRQ_HANDLED; + if (kbase_gpu_irq_handler(irq, data) == IRQ_HANDLED) + irq_state = IRQ_HANDLED; + + return irq_state; +} + static irq_handler_t kbase_handler_table[] = { [JOB_IRQ_TAG] = kbase_job_irq_handler, [MMU_IRQ_TAG] = kbase_mmu_irq_handler, [GPU_IRQ_TAG] = kbase_gpu_irq_handler, }; -#ifdef CONFIG_MALI_BIFROST_DEBUG -#define JOB_IRQ_HANDLER JOB_IRQ_TAG -#define GPU_IRQ_HANDLER GPU_IRQ_TAG - -/** - * kbase_gpu_irq_test_handler - Variant (for test) of kbase_gpu_irq_handler() - * @irq: IRQ number - * @data: Data associated with this IRQ (i.e. kbdev) - * @val: Value of the GPU_CONTROL_ENUM(GPU_IRQ_STATUS) - * - * Handle the GPU device interrupt source requests reflected in the - * given source bit-pattern. The test code caller is responsible for - * undertaking the required device power maintenace. - * - * Return: IRQ_HANDLED if the requests are from the GPU device, - * IRQ_NONE otherwise - */ -irqreturn_t kbase_gpu_irq_test_handler(int irq, void *data, u32 val) +irq_handler_t kbase_get_interrupt_handler(struct kbase_device *kbdev, u32 irq_tag) { - struct kbase_device *kbdev = kbase_untag(data); - - if (!val) - return IRQ_NONE; - - dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); - - kbase_gpu_interrupt(kbdev, val); - - return IRQ_HANDLED; + if (kbdev->nr_irqs == 1) + return kbase_combined_irq_handler; + else if (irq_tag < ARRAY_SIZE(kbase_handler_table)) + return kbase_handler_table[irq_tag]; + else + return NULL; } -KBASE_EXPORT_TEST_API(kbase_gpu_irq_test_handler); - -/** - * kbase_set_custom_irq_handler - Set a custom IRQ handler - * @kbdev: Device for which the handler is to be registered - * @custom_handler: Handler to be registered - * @irq_type: Interrupt type - * - * Registers given interrupt handler for requested interrupt type - * In the case where irq handler is not specified, the default handler shall be - * registered - * - * Return: 0 case success, error code otherwise - */ +#if IS_ENABLED(CONFIG_MALI_REAL_HW) +#ifdef CONFIG_MALI_BIFROST_DEBUG int kbase_set_custom_irq_handler(struct kbase_device *kbdev, irq_handler_t custom_handler, - int irq_type) + u32 irq_tag) { int result = 0; - irq_handler_t requested_irq_handler = NULL; + irq_handler_t handler = custom_handler; + const int irq = (kbdev->nr_irqs == 1) ? 0 : irq_tag; - KBASE_DEBUG_ASSERT((irq_type >= JOB_IRQ_HANDLER) && (irq_type <= GPU_IRQ_HANDLER)); + if (unlikely(!((irq_tag >= JOB_IRQ_TAG) && (irq_tag <= GPU_IRQ_TAG)))) { + dev_err(kbdev->dev, "Invalid irq_tag (%d)\n", irq_tag); + return -EINVAL; + } /* Release previous handler */ - if (kbdev->irqs[irq_type].irq) - free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type)); + if (kbdev->irqs[irq].irq) + free_irq(kbdev->irqs[irq].irq, kbase_tag(kbdev, irq)); - requested_irq_handler = (custom_handler != NULL) ? custom_handler : - kbase_handler_table[irq_type]; + /* If a custom handler isn't provided use the default handler */ + if (!handler) + handler = kbase_get_interrupt_handler(kbdev, irq_tag); - if (request_irq(kbdev->irqs[irq_type].irq, requested_irq_handler, - kbdev->irqs[irq_type].flags | IRQF_SHARED, dev_name(kbdev->dev), - kbase_tag(kbdev, irq_type)) != 0) { + if (request_irq(kbdev->irqs[irq].irq, handler, + kbdev->irqs[irq].flags | ((kbdev->nr_irqs == 1) ? 0 : IRQF_SHARED), + dev_name(kbdev->dev), kbase_tag(kbdev, irq)) != 0) { result = -EINVAL; - dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", - kbdev->irqs[irq_type].irq, irq_type); -#if IS_ENABLED(CONFIG_SPARSE_IRQ) - dev_err(kbdev->dev, - "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); -#endif /* CONFIG_SPARSE_IRQ */ + dev_err(kbdev->dev, "Can't request interrupt %u (index %u)\n", kbdev->irqs[irq].irq, + irq_tag); + if (IS_ENABLED(CONFIG_SPARSE_IRQ)) + dev_err(kbdev->dev, + "CONFIG_SPARSE_IRQ enabled - is the interrupt number correct for this config?\n"); } return result; @@ -325,30 +312,43 @@ static enum hrtimer_restart kbasep_test_interrupt_timeout(struct hrtimer *timer) return HRTIMER_NORESTART; } -static int kbasep_common_test_interrupt(struct kbase_device *const kbdev, u32 tag) +/** + * validate_interrupt - Validate an interrupt + * @kbdev: Kbase device + * @tag: Tag to choose the interrupt + * + * To validate the settings for the interrupt, write a value on RAWSTAT + * register to trigger interrupt. Then with custom interrupt handler + * check whether the interrupt happens within reasonable time. + * + * Return: 0 if validating interrupt succeeds. + */ +static int validate_interrupt(struct kbase_device *const kbdev, u32 tag) { int err = 0; - irq_handler_t test_handler; - + irq_handler_t handler; + const int irq = (kbdev->nr_irqs == 1) ? 0 : tag; u32 old_mask_val; u16 mask_offset; u16 rawstat_offset; switch (tag) { case JOB_IRQ_TAG: - test_handler = kbase_job_irq_test_handler; + handler = kbase_job_irq_test_handler; rawstat_offset = JOB_CONTROL_ENUM(JOB_IRQ_RAWSTAT); mask_offset = JOB_CONTROL_ENUM(JOB_IRQ_MASK); break; case MMU_IRQ_TAG: - test_handler = kbase_mmu_irq_test_handler; + handler = kbase_mmu_irq_test_handler; rawstat_offset = MMU_CONTROL_ENUM(IRQ_RAWSTAT); mask_offset = MMU_CONTROL_ENUM(IRQ_MASK); break; case GPU_IRQ_TAG: /* already tested by pm_driver - bail out */ - default: return 0; + default: + dev_err(kbdev->dev, "Invalid tag (%d)\n", tag); + return -EINVAL; } /* store old mask */ @@ -356,9 +356,9 @@ static int kbasep_common_test_interrupt(struct kbase_device *const kbdev, u32 ta /* mask interrupts */ kbase_reg_write32(kbdev, mask_offset, 0x0); - if (kbdev->irqs[tag].irq) { + if (kbdev->irqs[irq].irq) { /* release original handler and install test handler */ - if (kbase_set_custom_irq_handler(kbdev, test_handler, tag) != 0) { + if (kbase_set_custom_irq_handler(kbdev, handler, tag) != 0) { err = -EINVAL; } else { kbasep_irq_test_data.timeout = 0; @@ -376,12 +376,12 @@ static int kbasep_common_test_interrupt(struct kbase_device *const kbdev, u32 ta wait_event(kbasep_irq_test_data.wait, kbasep_irq_test_data.triggered != 0); if (kbasep_irq_test_data.timeout != 0) { - dev_err(kbdev->dev, "Interrupt %d (index %d) didn't reach CPU.\n", - kbdev->irqs[tag].irq, tag); + dev_err(kbdev->dev, "Interrupt %u (index %u) didn't reach CPU.\n", + kbdev->irqs[irq].irq, irq); err = -EINVAL; } else { - dev_dbg(kbdev->dev, "Interrupt %d (index %d) reached CPU.\n", - kbdev->irqs[tag].irq, tag); + dev_dbg(kbdev->dev, "Interrupt %u (index %u) reached CPU.\n", + kbdev->irqs[irq].irq, irq); } hrtimer_cancel(&kbasep_irq_test_data.timer); @@ -391,15 +391,15 @@ static int kbasep_common_test_interrupt(struct kbase_device *const kbdev, u32 ta kbase_reg_write32(kbdev, mask_offset, 0x0); /* release test handler */ - free_irq(kbdev->irqs[tag].irq, kbase_tag(kbdev, tag)); + free_irq(kbdev->irqs[irq].irq, kbase_tag(kbdev, irq)); } /* restore original interrupt */ - if (request_irq(kbdev->irqs[tag].irq, kbase_handler_table[tag], - kbdev->irqs[tag].flags | IRQF_SHARED, dev_name(kbdev->dev), - kbase_tag(kbdev, tag))) { - dev_err(kbdev->dev, "Can't restore original interrupt %d (index %d)\n", - kbdev->irqs[tag].irq, tag); + if (request_irq(kbdev->irqs[irq].irq, kbase_get_interrupt_handler(kbdev, tag), + kbdev->irqs[irq].flags | ((kbdev->nr_irqs == 1) ? 0 : IRQF_SHARED), + dev_name(kbdev->dev), kbase_tag(kbdev, irq))) { + dev_err(kbdev->dev, "Can't restore original interrupt %u (index %u)\n", + kbdev->irqs[irq].irq, tag); err = -EINVAL; } } @@ -409,7 +409,8 @@ static int kbasep_common_test_interrupt(struct kbase_device *const kbdev, u32 ta return err; } -int kbasep_common_test_interrupt_handlers(struct kbase_device *const kbdev) +#if IS_ENABLED(CONFIG_MALI_REAL_HW) +int kbase_validate_interrupts(struct kbase_device *const kbdev) { int err; @@ -419,14 +420,14 @@ int kbasep_common_test_interrupt_handlers(struct kbase_device *const kbdev) /* A suspend won't happen during startup/insmod */ kbase_pm_context_active(kbdev); - err = kbasep_common_test_interrupt(kbdev, JOB_IRQ_TAG); + err = validate_interrupt(kbdev, JOB_IRQ_TAG); if (err) { dev_err(kbdev->dev, "Interrupt JOB_IRQ didn't reach CPU. Check interrupt assignments.\n"); goto out; } - err = kbasep_common_test_interrupt(kbdev, MMU_IRQ_TAG); + err = validate_interrupt(kbdev, MMU_IRQ_TAG); if (err) { dev_err(kbdev->dev, "Interrupt MMU_IRQ didn't reach CPU. Check interrupt assignments.\n"); @@ -440,25 +441,21 @@ out: return err; } +#endif /* CONFIG_MALI_REAL_HW */ #endif /* CONFIG_MALI_BIFROST_DEBUG */ int kbase_install_interrupts(struct kbase_device *kbdev) { - u32 nr = ARRAY_SIZE(kbase_handler_table); - int err; u32 i; - for (i = 0; i < nr; i++) { - err = request_irq(kbdev->irqs[i].irq, kbase_handler_table[i], - kbdev->irqs[i].flags | IRQF_SHARED, dev_name(kbdev->dev), - kbase_tag(kbdev, i)); - if (err) { - dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", + for (i = 0; i < kbdev->nr_irqs; i++) { + const int result = request_irq( + kbdev->irqs[i].irq, kbase_get_interrupt_handler(kbdev, i), + kbdev->irqs[i].flags | ((kbdev->nr_irqs == 1) ? 0 : IRQF_SHARED), + dev_name(kbdev->dev), kbase_tag(kbdev, i)); + if (result) { + dev_err(kbdev->dev, "Can't request interrupt %u (index %u)\n", kbdev->irqs[i].irq, i); -#if IS_ENABLED(CONFIG_SPARSE_IRQ) - dev_err(kbdev->dev, - "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); -#endif /* CONFIG_SPARSE_IRQ */ goto release; } } @@ -466,18 +463,21 @@ int kbase_install_interrupts(struct kbase_device *kbdev) return 0; release: + if (IS_ENABLED(CONFIG_SPARSE_IRQ)) + dev_err(kbdev->dev, + "CONFIG_SPARSE_IRQ enabled - is the interrupt number correct for this config?\n"); + while (i-- > 0) free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); - return err; + return -EINVAL; } void kbase_release_interrupts(struct kbase_device *kbdev) { - u32 nr = ARRAY_SIZE(kbase_handler_table); u32 i; - for (i = 0; i < nr; i++) { + for (i = 0; i < kbdev->nr_irqs; i++) { if (kbdev->irqs[i].irq) free_irq(kbdev->irqs[i].irq, kbase_tag(kbdev, i)); } @@ -485,10 +485,9 @@ void kbase_release_interrupts(struct kbase_device *kbdev) void kbase_synchronize_irqs(struct kbase_device *kbdev) { - u32 nr = ARRAY_SIZE(kbase_handler_table); u32 i; - for (i = 0; i < nr; i++) { + for (i = 0; i < kbdev->nr_irqs; i++) { if (kbdev->irqs[i].irq) synchronize_irq(kbdev->irqs[i].irq); } diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c index 8f06058bbdb4..e822dc59977b 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c @@ -177,17 +177,14 @@ static u64 select_job_chain(struct kbase_jd_atom *katom) static inline bool kbasep_jm_wait_js_free(struct kbase_device *kbdev, unsigned int js, struct kbase_context *kctx) { - const ktime_t wait_loop_start = ktime_get_raw(); - const s64 max_timeout = (s64)kbdev->js_data.js_free_wait_time_ms; - s64 diff = 0; - + u32 val; + const u32 timeout_us = kbdev->js_data.js_free_wait_time_ms * USEC_PER_MSEC; /* wait for the JS_COMMAND_NEXT register to reach the given status value */ - do { - if (!kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT))) - return true; + const int err = kbase_reg_poll32_timeout(kbdev, JOB_SLOT_OFFSET(js, COMMAND_NEXT), val, + !val, 0, timeout_us, false); - diff = ktime_to_ms(ktime_sub(ktime_get_raw(), wait_loop_start)); - } while (diff < max_timeout); + if (!err) + return true; dev_err(kbdev->dev, "Timeout in waiting for job slot %u to become free for ctx %d_%u", js, kctx->tgid, kctx->id); @@ -221,7 +218,7 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, /* start MMU, medium priority, cache clean/flush on end, clean/flush on * start */ - cfg = kctx->as_nr; + cfg = (u32)kctx->as_nr; if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION) && !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) @@ -386,10 +383,10 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) /* Note: This is inherently unfair, as we always check for lower * numbered interrupts before the higher numbered ones. */ - i = ffs(finished) - 1; + i = (unsigned int)ffs((int)finished) - 1u; do { - int nr_done; + u32 nr_done; u32 active; u32 completion_code = BASE_JD_EVENT_DONE; /* assume OK */ u64 job_tail = 0; @@ -453,7 +450,7 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) } kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_CLEAR), - done & ((1 << i) | (1 << (i + 16)))); + done & ((1u << i) | (1u << (i + 16)))); active = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_JS_STATE)); if (((active >> i) & 1) == 0 && (((done >> (i + 16)) & 1) == 0)) { @@ -514,8 +511,8 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) nr_done -= (active >> i) & 1; nr_done -= (active >> (i + 16)) & 1; - if (nr_done <= 0) { - dev_warn(kbdev->dev, "Spurious interrupt on slot %d", i); + if (nr_done == 0 || nr_done > SLOT_RB_SIZE) { + dev_warn(kbdev->dev, "Spurious interrupt on slot %u", i); goto spurious; } @@ -556,7 +553,7 @@ spurious: finished = (done & 0xFFFF) | failed; if (done) end_timestamp = ktime_get_raw(); - } while (finished & (1 << i)); + } while (finished & (1u << i)); kbasep_job_slot_update_head_start_timestamp(kbdev, i, end_timestamp); } @@ -580,7 +577,7 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, uns u64 job_in_head_before; u32 status_reg_after; - WARN_ON(action & (~JS_COMMAND_MASK)); + WARN_ON(action & (~(u32)JS_COMMAND_MASK)); /* Check the head pointer */ job_in_head_before = kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(js, HEAD)); @@ -808,11 +805,12 @@ void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) struct kbase_device *kbdev = kctx->kbdev; unsigned long timeout = msecs_to_jiffies(ZAP_TIMEOUT); - timeout = wait_event_timeout(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0, timeout); + timeout = wait_event_timeout(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0, + (long)timeout); if (timeout != 0) timeout = wait_event_timeout(kctx->jctx.sched_info.ctx.is_scheduled_wait, - !kbase_ctx_flag(kctx, KCTX_SCHEDULED), timeout); + !kbase_ctx_flag(kctx, KCTX_SCHEDULED), (long)timeout); /* Neither wait timed out; all done! */ if (timeout != 0) @@ -899,7 +897,8 @@ void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, unsigned int js JS_COMMAND_SOFT_STOP | sw_flags); } -void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, struct kbase_jd_atom *target_katom) +void kbase_job_slot_softstop(struct kbase_device *kbdev, unsigned int js, + struct kbase_jd_atom *target_katom) { kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u); } @@ -979,7 +978,7 @@ void kbase_reset_gpu_assert_failed_or_prevented(struct kbase_device *kbdev) static void kbase_debug_dump_registers(struct kbase_device *kbdev) { - int i; + unsigned int i; kbase_io_history_dump(kbdev); @@ -991,7 +990,7 @@ static void kbase_debug_dump_registers(struct kbase_device *kbdev) kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_RAWSTAT)), kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_JS_STATE))); for (i = 0; i < 3; i++) { - dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD=0x%016llx", i, + dev_err(kbdev->dev, " JS%u_STATUS=0x%08x JS%u_HEAD=0x%016llx", i, kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(i, STATUS)), i, kbase_reg_read64(kbdev, JOB_SLOT_OFFSET(i, HEAD))); } @@ -1020,7 +1019,6 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) ktime_t end_timestamp = ktime_get_raw(); struct kbasep_js_device_data *js_devdata; bool silent = false; - u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; kbdev = container_of(data, struct kbase_device, hwaccess.backend.reset_work); @@ -1088,13 +1086,15 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) kbdev->irq_reset_flush = false; if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { - /* Ensure that L2 is not transitioning when we send the reset - * command - */ - while (--max_loops && kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) - ; + u64 val; + const u32 timeout_us = + kbase_get_timeout_ms(kbdev, KBASE_CLEAN_CACHE_TIMEOUT) * USEC_PER_MSEC; + /* Ensure that L2 is not transitioning when we send the reset command */ + const int err = read_poll_timeout_atomic(kbase_pm_get_trans_cores, val, !val, 0, + timeout_us, false, kbdev, + KBASE_PM_CORE_L2); - WARN(!max_loops, "L2 power transition timed out while trying to reset\n"); + WARN(err, "L2 power transition timed out while trying to reset\n"); } mutex_lock(&kbdev->pm.lock); @@ -1212,7 +1212,7 @@ static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer) static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) { unsigned int i; - int pending_jobs = 0; + u32 pending_jobs = 0; /* Count the number of jobs */ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) @@ -1266,7 +1266,7 @@ static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) */ bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev, unsigned int flags) { - int i; + unsigned int i; #ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_gpu_lost(kbdev)) { diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c index 50cf19d876c5..842209f9c049 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c @@ -159,9 +159,9 @@ bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) return false; } -int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, unsigned int js) +u32 kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, unsigned int js) { - int nr = 0; + u32 nr = 0; int i; lockdep_assert_held(&kbdev->hwaccess_lock); @@ -539,7 +539,7 @@ static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) } static int kbase_jm_protected_entry(struct kbase_device *kbdev, struct kbase_jd_atom **katom, - int idx, int js) + int idx, unsigned int js) { int err = 0; @@ -594,7 +594,7 @@ static int kbase_jm_protected_entry(struct kbase_device *kbdev, struct kbase_jd_ } static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbase_jd_atom **katom, - int idx, int js) + int idx, unsigned int js) { int err = 0; @@ -758,7 +758,7 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbas } static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, struct kbase_jd_atom **katom, - int idx, int js) + int idx, unsigned int js) { int err = 0; @@ -1274,7 +1274,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp struct kbasep_js_device_data *js_devdata = &kbdev->js_data; unsigned int i; - if (!kbase_ctx_flag(katom->kctx, KCTX_DYING)) { + if (!kbase_ctx_flag(katom->kctx, KCTX_DYING) && + !kbase_ctx_flag(katom->kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) { dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", js, completion_code, kbase_gpu_exception_name(completion_code)); @@ -1383,7 +1384,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp /* Check if there are lower priority jobs to soft stop */ kbase_job_slot_ctx_priority_check_locked(kctx, katom); - kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); + kbase_jm_try_kick(kbdev, 1UL << katom->slot_nr); } /* For partial shader core off L2 cache flush */ @@ -1535,7 +1536,7 @@ static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) if (dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_RETURN_TO_JS) - return dep_atom->slot_nr; + return (int)dep_atom->slot_nr; } return -1; } @@ -1714,10 +1715,12 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, struct kbase_ } if (stop_x_dep_idx0 != -1) - kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0, NULL, action); + kbase_backend_soft_hard_stop_slot(kbdev, kctx, (unsigned int)stop_x_dep_idx0, NULL, + action); if (stop_x_dep_idx1 != -1) - kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1, NULL, action); + kbase_backend_soft_hard_stop_slot(kbdev, kctx, (unsigned int)stop_x_dep_idx1, NULL, + action); return ret; } diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c index c40ffbf9e089..202671b323d5 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c @@ -174,12 +174,12 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) * now. Hard stop the slot. */ #if !KBASE_DISABLE_SCHEDULING_HARD_STOPS - int ms = js_devdata->scheduling_period_ns / 1000000u; + u32 ms = js_devdata->scheduling_period_ns / 1000000u; if (!kbase_is_quick_reset_enabled(kbdev)) dev_warn( kbdev->dev, - "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", - (unsigned long)ticks, (unsigned long)ms); + "JS: Job Hard-Stopped (took more than %u ticks at %u ms/tick)", + ticks, ms); kbase_job_slot_hardstop(atom->kctx, s, atom); #endif } else if (ticks == gpu_reset_ticks) { @@ -210,11 +210,11 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) * ticks. Hard stop the slot. */ #if !KBASE_DISABLE_SCHEDULING_HARD_STOPS - int ms = js_devdata->scheduling_period_ns / 1000000u; + u32 ms = js_devdata->scheduling_period_ns / 1000000u; dev_warn( kbdev->dev, - "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", - (unsigned long)ticks, (unsigned long)ms); + "JS: Job Hard-Stopped (took more than %u ticks at %u ms/tick)", + ticks, ms); kbase_job_slot_hardstop(atom->kctx, s, atom); #endif } else if (ticks == js_devdata->gpu_reset_ticks_dumping) { diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c index 4c6bb912105e..41b9b37797d3 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c @@ -68,6 +68,8 @@ #include #include +#include + #if MALI_USE_CSF #include @@ -78,7 +80,7 @@ /* Array for storing the value of SELECT register for each type of core */ static u64 ipa_ctl_select_config[KBASE_IPA_CORE_TYPE_NUM]; -static bool ipa_control_timer_enabled; +static u32 ipa_control_timer_enabled; #endif #if MALI_USE_CSF @@ -142,8 +144,8 @@ struct control_reg_values_t { struct job_slot { int job_active; int job_queued; - int job_complete_irq_asserted; - int job_irq_mask; + u32 job_complete_irq_asserted; + u32 job_irq_mask; int job_disabled; }; @@ -512,7 +514,7 @@ void *gpu_device_get_data(void *model) return dummy->kbdev; } -#define signal_int(m, s) m->slots[(s)].job_complete_irq_asserted = 1 +#define signal_int(m, s) m->slots[(s)].job_complete_irq_asserted = 1u static char *no_mali_gpu = CONFIG_MALI_NO_MALI_DEFAULT_GPU; module_param(no_mali_gpu, charp, 0000); @@ -650,7 +652,7 @@ static void gpu_model_dump_prfcnt_blocks(u64 *values, u32 *out_index, u32 block_ for (block_idx = 0; block_idx < block_count; block_idx++) { /* only dump values if core is present */ - if (!(blocks_present & (1 << block_idx))) { + if (!(blocks_present & (1U << block_idx))) { #if MALI_USE_CSF /* if CSF dump zeroed out block */ memset(&prfcnt_base[*out_index], 0, KBASE_DUMMY_MODEL_BLOCK_SIZE); @@ -712,6 +714,11 @@ static void gpu_model_dump_nolock(void) performance_counters.time += 10; } +static void gpu_model_raise_irq(void *model, u32 irq) +{ + gpu_device_raise_irq(model, irq); +} + #if !MALI_USE_CSF static void midgard_model_dump_prfcnt(void) { @@ -743,7 +750,7 @@ void gpu_model_glb_request_job_irq(void *model) spin_lock_irqsave(&hw_error_status.access_lock, flags); hw_error_status.job_irq_status |= JOB_IRQ_GLOBAL_IF; spin_unlock_irqrestore(&hw_error_status.access_lock, flags); - gpu_device_raise_irq(model, MODEL_LINUX_JOB_IRQ); + gpu_model_raise_irq(model, MODEL_LINUX_JOB_IRQ); } #endif /* !MALI_USE_CSF */ @@ -767,13 +774,13 @@ static void init_register_statuses(struct dummy_model_t *dummy) for (i = 0; i < NUM_MMU_AS; i++) { hw_error_status.as_command[i] = 0; hw_error_status.as_faultstatus[i] = 0; - hw_error_status.mmu_irq_mask |= 1 << i; + hw_error_status.mmu_irq_mask |= (1u << i); } performance_counters.time = 0; } -static void update_register_statuses(struct dummy_model_t *dummy, unsigned int job_slot) +static void update_register_statuses(struct dummy_model_t *dummy, u32 job_slot) { lockdep_assert_held(&hw_error_status.access_lock); @@ -915,7 +922,7 @@ static void update_register_statuses(struct dummy_model_t *dummy, unsigned int j } /* end of job register statuses */ if (hw_error_status.errors_mask & IS_A_MMU_ERROR) { - int i; + u32 i; for (i = 0; i < NUM_MMU_AS; i++) { if (i == hw_error_status.faulty_mmu_as) { @@ -965,9 +972,10 @@ static void update_register_statuses(struct dummy_model_t *dummy, unsigned int j if (hw_error_status.errors_mask & KBASE_TRANSTAB_BUS_FAULT) hw_error_status.mmu_irq_rawstat |= - 1 << (16 + i); /* bus error */ + 1u << (16 + i); /* bus error */ else - hw_error_status.mmu_irq_rawstat |= 1 << i; /* page fault */ + hw_error_status.mmu_irq_rawstat |= + (1u << i); /* page fault */ } } } /*end of mmu register statuses */ @@ -979,11 +987,11 @@ static void update_register_statuses(struct dummy_model_t *dummy, unsigned int j hw_error_status.gpu_error_irq |= 1; switch (hw_error_status.errors_mask & IS_A_GPU_ERROR) { case KBASE_DELAYED_BUS_FAULT: - hw_error_status.gpu_fault_status = (1 << 7); + hw_error_status.gpu_fault_status = (1u << 7); break; case KBASE_SHAREABILITY_FAULT: - hw_error_status.gpu_fault_status = (1 << 7) | (1 << 3); + hw_error_status.gpu_fault_status = (1u << 7) | (1u << 3); break; default: @@ -1119,7 +1127,7 @@ static void midgard_model_get_outputs(void *h) lockdep_assert_held(&hw_error_status.access_lock); if (hw_error_status.job_irq_status) - gpu_device_raise_irq(dummy, MODEL_LINUX_JOB_IRQ); + gpu_model_raise_irq(dummy, MODEL_LINUX_JOB_IRQ); if ((dummy->power_changed && dummy->power_changed_mask) || (dummy->reset_completed & dummy->reset_completed_mask) || @@ -1130,16 +1138,16 @@ static void midgard_model_get_outputs(void *h) (dummy->flush_pa_range_completed && dummy->flush_pa_range_completed_irq_enabled) || #endif (dummy->clean_caches_completed && dummy->clean_caches_completed_irq_enabled)) - gpu_device_raise_irq(dummy, MODEL_LINUX_GPU_IRQ); + gpu_model_raise_irq(dummy, MODEL_LINUX_GPU_IRQ); if (hw_error_status.mmu_irq_rawstat & hw_error_status.mmu_irq_mask) - gpu_device_raise_irq(dummy, MODEL_LINUX_MMU_IRQ); + gpu_model_raise_irq(dummy, MODEL_LINUX_MMU_IRQ); } static void midgard_model_update(void *h) { struct dummy_model_t *dummy = (struct dummy_model_t *)h; - int i; + u32 i; lockdep_assert_held(&hw_error_status.access_lock); @@ -1157,8 +1165,8 @@ static void midgard_model_update(void *h) * as we will overwrite the register status of the job in * the head registers - which has not yet been read */ - if ((hw_error_status.job_irq_rawstat & (1 << (i + 16))) || - (hw_error_status.job_irq_rawstat & (1 << i))) { + if ((hw_error_status.job_irq_rawstat & (1u << (i + 16))) || + (hw_error_status.job_irq_rawstat & (1u << i))) { continue; } @@ -1169,7 +1177,7 @@ static void midgard_model_update(void *h) #endif /* CONFIG_MALI_BIFROST_ERROR_INJECT */ update_register_statuses(dummy, i); /*if this job slot returned failures we cannot use it */ - if (hw_error_status.job_irq_rawstat & (1 << (i + 16))) { + if (hw_error_status.job_irq_rawstat & (1u << (i + 16))) { dummy->slots[i].job_active = 0; continue; } @@ -1177,7 +1185,7 @@ static void midgard_model_update(void *h) dummy->slots[i].job_active = dummy->slots[i].job_queued; dummy->slots[i].job_queued = 0; if (dummy->slots[i].job_active) { - if (hw_error_status.job_irq_rawstat & (1 << (i + 16))) + if (hw_error_status.job_irq_rawstat & (1u << (i + 16))) model_error_log(KBASE_CORE, "\natom %lld running a job on a dirty slot", hw_error_status.current_jc); @@ -1193,7 +1201,7 @@ static void invalidate_active_jobs(struct dummy_model_t *dummy) for (i = 0; i < NUM_SLOTS; i++) { if (dummy->slots[i].job_active) { - hw_error_status.job_irq_rawstat |= (1 << (16 + i)); + hw_error_status.job_irq_rawstat |= (1u << (16 + i)); hw_error_status.js_status[i] = 0x7f; /*UNKNOWN*/ } @@ -1256,7 +1264,7 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) int i; for (i = 0; i < NUM_SLOTS; i++) { - if (value & ((1 << i) | (1 << (i + 16)))) + if (value & ((1u << i) | (1u << (i + 16)))) dummy->slots[i].job_complete_irq_asserted = 0; /* hw_error_status.js_status[i] is cleared in * update_job_irq_js_state @@ -1279,6 +1287,9 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) hw_error_status.job_irq_rawstat &= ~(value); hw_error_status.job_irq_status &= ~(value); + } else if (addr == JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)) { + hw_error_status.job_irq_rawstat |= value; + hw_error_status.job_irq_status |= value; } else if (addr == JOB_CONTROL_REG(JOB_IRQ_MASK)) { /* ignore JOB_IRQ_MASK as it is handled by CSFFW */ #endif /* !MALI_USE_CSF */ @@ -1378,7 +1389,7 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) pr_debug("Received IPA_CONTROL command"); } else if (addr == IPA_CONTROL_REG(TIMER) ) { - ipa_control_timer_enabled = value ? true : false; + ipa_control_timer_enabled = value ? 1U : 0U; } else if ((addr >= IPA_CONTROL_REG(SELECT_CSHW_LO)) && (addr <= IPA_CONTROL_REG(SELECT_SHADER_HI))) { enum kbase_ipa_core_type core_type = @@ -1399,7 +1410,7 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value) hw_error_status.mmu_irq_rawstat &= (~value); } else if ((addr >= MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) && (addr <= MMU_STAGE1_REG(MMU_AS_REG(15, AS_STATUS)))) { - int mem_addr_space = (addr - MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) >> 6; + u32 mem_addr_space = (addr - MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) >> 6; switch (addr & 0x3F) { case AS_COMMAND: @@ -1622,7 +1633,7 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) #if MALI_USE_CSF ((dummy->flush_pa_range_completed_irq_enabled ? 1u : 0u) << 20) | #endif - (dummy->power_changed_mask << 9) | (1 << 7) | 1; + (dummy->power_changed_mask << 9) | (1u << 7) | 1u; pr_debug("GPU_IRQ_MASK read %x", *value); } else if (addr == GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) { *value = ((dummy->clean_caches_completed ? 1u : 0u) << 17) | @@ -1891,7 +1902,7 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) *value = 0; } else if (addr >= MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO)) && addr <= MMU_STAGE1_REG(MMU_AS_REG(15, AS_STATUS))) { - int mem_addr_space = (addr - MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) >> 6; + u32 mem_addr_space = (addr - MMU_STAGE1_REG(MMU_AS_REG(0, AS_TRANSTAB_LO))) >> 6; switch (addr & 0x3F) { case AS_TRANSTAB_LO: @@ -1927,7 +1938,7 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) default: model_error_log( KBASE_CORE, - "Dummy model register access: Reading unsupported MMU #%d register 0x%x. Returning 0\n", + "Dummy model register access: Reading unsupported MMU #%u register 0x%x. Returning 0\n", mem_addr_space, addr); *value = 0; break; @@ -2155,3 +2166,9 @@ int gpu_model_control(void *model, struct kbase_model_control_params *params) return 0; } + +u64 midgard_model_arch_timer_get_cntfrq(void *h) +{ + CSTD_UNUSED(h); + return arch_timer_get_cntfrq(); +} diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h index 3c6561a2f7b9..d4d2160a4438 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h @@ -134,10 +134,10 @@ struct error_status_t { u32 errors_mask; u32 mmu_table_level; - int faulty_mmu_as; + u32 faulty_mmu_as; u64 current_jc; - int current_job_slot; + u32 current_job_slot; u32 job_irq_rawstat; u32 job_irq_status; @@ -168,7 +168,7 @@ struct gpu_model_prfcnt_en { u32 shader; }; -void midgard_set_error(int job_slot); +void midgard_set_error(u32 job_slot); int job_atom_inject_error(struct kbase_error_params *params); int gpu_model_control(void *h, struct kbase_model_control_params *params); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_error_generator.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_error_generator.c index 072ad5bb01a0..86d4e26bd6b4 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_error_generator.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_error_generator.c @@ -131,7 +131,7 @@ int job_atom_inject_error(struct kbase_error_params *params) return 0; } -void midgard_set_error(int job_slot) +void midgard_set_error(u32 job_slot) { #ifdef CONFIG_MALI_ERROR_INJECT_RANDOM gpu_generate_error(); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.c index 098b60d4f4a2..fa12e52143f2 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.c @@ -37,68 +37,39 @@ struct model_irq_data { struct work_struct work; }; -static void serve_job_irq(struct work_struct *work) -{ - struct model_irq_data *data = container_of(work, struct model_irq_data, work); - struct kbase_device *kbdev = data->kbdev; - - /* Make sure no worker is already serving this IRQ */ - while (atomic_cmpxchg(&kbdev->serving_job_irq, 1, 0) == 1) { - u32 val; - - while ((val = kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_STATUS)))) { - unsigned long flags; - - /* Handle the IRQ */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -#if MALI_USE_CSF - kbase_csf_interrupt(kbdev, val); -#else - kbase_job_done(kbdev, val); -#endif - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } +#define DEFINE_SERVE_IRQ(irq_handler) \ + static void serve_##irq_handler(struct work_struct *work) \ + { \ + struct model_irq_data *data = container_of(work, struct model_irq_data, work); \ + struct kbase_device *kbdev = data->kbdev; \ + irq_handler(kbdev); \ + kmem_cache_free(kbdev->irq_slab, data); \ } - kmem_cache_free(kbdev->irq_slab, data); -} - -static void serve_gpu_irq(struct work_struct *work) +static void job_irq(struct kbase_device *kbdev) { - struct model_irq_data *data = container_of(work, struct model_irq_data, work); - struct kbase_device *kbdev = data->kbdev; - /* Make sure no worker is already serving this IRQ */ - while (atomic_cmpxchg(&kbdev->serving_gpu_irq, 1, 0) == 1) { - u32 val; - - while ((val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_STATUS)))) { - /* Handle the GPU_IRQ */ - kbase_gpu_interrupt(kbdev, val); - } - - } - - kmem_cache_free(kbdev->irq_slab, data); + while (atomic_cmpxchg(&kbdev->serving_job_irq, 1, 0) == 1) + kbase_get_interrupt_handler(kbdev, JOB_IRQ_TAG)(0, kbdev); } +DEFINE_SERVE_IRQ(job_irq) -static void serve_mmu_irq(struct work_struct *work) +static void gpu_irq(struct kbase_device *kbdev) { - struct model_irq_data *data = container_of(work, struct model_irq_data, work); - struct kbase_device *kbdev = data->kbdev; - /* Make sure no worker is already serving this IRQ */ - if (atomic_cmpxchg(&kbdev->serving_mmu_irq, 1, 0) == 1) { - u32 val; - - while ((val = kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_STATUS)))) { - /* Handle the IRQ */ - kbase_mmu_interrupt(kbdev, val); - } - } - - kmem_cache_free(kbdev->irq_slab, data); + while (atomic_cmpxchg(&kbdev->serving_gpu_irq, 1, 0) == 1) + kbase_get_interrupt_handler(kbdev, GPU_IRQ_TAG)(0, kbdev); } +DEFINE_SERVE_IRQ(gpu_irq) + +static void mmu_irq(struct kbase_device *kbdev) +{ + /* Make sure no worker is already serving this IRQ */ + while (atomic_cmpxchg(&kbdev->serving_mmu_irq, 1, 0) == 1) + kbase_get_interrupt_handler(kbdev, MMU_IRQ_TAG)(0, kbdev); +} +DEFINE_SERVE_IRQ(mmu_irq) + void gpu_device_raise_irq(void *model, u32 irq) { @@ -156,6 +127,9 @@ int kbase_install_interrupts(struct kbase_device *kbdev) return -ENOMEM; } + kbdev->nr_irqs = 3; + + return 0; } @@ -175,23 +149,13 @@ void kbase_synchronize_irqs(struct kbase_device *kbdev) KBASE_EXPORT_TEST_API(kbase_synchronize_irqs); int kbase_set_custom_irq_handler(struct kbase_device *kbdev, irq_handler_t custom_handler, - int irq_type) + u32 irq_tag) { return 0; } KBASE_EXPORT_TEST_API(kbase_set_custom_irq_handler); -irqreturn_t kbase_gpu_irq_test_handler(int irq, void *data, u32 val) -{ - if (!val) - return IRQ_NONE; - - return IRQ_HANDLED; -} - -KBASE_EXPORT_TEST_API(kbase_gpu_irq_test_handler); - int kbase_gpu_device_create(struct kbase_device *kbdev) { kbdev->model = midgard_model_create(kbdev); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h index 8f09afe3d1cc..77e089ef45c8 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -116,6 +116,15 @@ void midgard_model_write_reg(void *h, u32 addr, u32 value); */ void midgard_model_read_reg(void *h, u32 addr, u32 *const value); +/** + * midgard_model_arch_timer_get_cntfrq - Get Model specific System Timer Frequency + * + * @h: Model handle. + * + * Return: Frequency in Hz + */ +u64 midgard_model_arch_timer_get_cntfrq(void *h); + /** * gpu_device_raise_irq() - Private IRQ raise function. * diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c index 61b756855060..ca4e73d3fbb7 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c @@ -33,6 +33,7 @@ #include #include #else +#include #include #include #endif /* !MALI_USE_CSF */ @@ -42,6 +43,7 @@ #include #include + static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data); static void kbase_pm_hwcnt_disable_worker(struct work_struct *data); static void kbase_pm_gpu_clock_control_worker(struct work_struct *data); @@ -51,6 +53,8 @@ int kbase_pm_runtime_init(struct kbase_device *kbdev) struct kbase_pm_callback_conf *callbacks; callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS; + + if (callbacks) { kbdev->pm.backend.callback_power_on = callbacks->power_on_callback; kbdev->pm.backend.callback_power_off = callbacks->power_off_callback; @@ -178,6 +182,10 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) !kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_1997) && kbdev->pm.backend.callback_power_runtime_gpu_active && kbdev->pm.backend.callback_power_runtime_gpu_idle; + + kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa = + kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TITANHW_2938) && + kbdev->pm.backend.gpu_sleep_supported; #endif if (IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED)) @@ -239,6 +247,58 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) */ } +#if MALI_USE_CSF +static bool wait_cond_mmu_fault_handling_in_gpu_poweroff_wait_wq(struct kbase_device *kbdev, + int faults_pending) +{ + struct kbase_pm_backend_data *backend = &kbdev->pm.backend; + bool cond = false; + + kbase_pm_lock(kbdev); + cond = backend->poweron_required || (faults_pending == 0); + kbase_pm_unlock(kbdev); + + return cond; +} +#endif + +static void wait_for_mmu_fault_handling_in_gpu_poweroff_wait_wq(struct kbase_device *kbdev) +{ +#if MALI_USE_CSF + bool reset_triggered = false; + int ret = 0; + + lockdep_assert_held(&kbdev->pm.lock); + + do { + const u64 timeout_us = kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT) * USEC_PER_MSEC; + const unsigned long delay_us = 10; + int faults_pending = 0; + + kbase_pm_unlock(kbdev); + ret = read_poll_timeout_atomic( + atomic_read, faults_pending, + wait_cond_mmu_fault_handling_in_gpu_poweroff_wait_wq(kbdev, faults_pending), + delay_us, timeout_us, false, &kbdev->faults_pending); + kbase_pm_lock(kbdev); + + if (ret && !reset_triggered) { + dev_err(kbdev->dev, + "Wait for fault handling timed-out in gpu_poweroff_wait_wq"); + if (kbase_prepare_to_reset_gpu(kbdev, + RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) { + kbase_reset_gpu(kbdev); + reset_triggered = true; + } + } + } while (ret); +#else + kbase_pm_unlock(kbdev); + kbase_flush_mmu_wqs(kbdev); + kbase_pm_lock(kbdev); +#endif +} + static void pm_handle_power_off(struct kbase_device *kbdev) { struct kbase_pm_backend_data *backend = &kbdev->pm.backend; @@ -283,9 +343,7 @@ static void pm_handle_power_off(struct kbase_device *kbdev) * process. Interrupts are disabled so no more faults * should be generated at this point. */ - kbase_pm_unlock(kbdev); - kbase_flush_mmu_wqs(kbdev); - kbase_pm_lock(kbdev); + wait_for_mmu_fault_handling_in_gpu_poweroff_wait_wq(kbdev); #ifdef CONFIG_MALI_ARBITER_SUPPORT /* poweron_required may have changed while pm lock @@ -698,12 +756,6 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, unsigned int flags) kbdev->pm.backend.gpu_cycle_counter_requests = 0; spin_unlock_irqrestore(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, irq_flags); - /* We are ready to receive IRQ's now as power policy is set up, so - * enable them now. - */ -#ifdef CONFIG_MALI_BIFROST_DEBUG - kbdev->pm.backend.driver_ready_for_irqs = true; -#endif kbase_pm_enable_interrupts(kbdev); WARN_ON(!kbdev->pm.backend.gpu_powered); @@ -910,7 +962,9 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) { unsigned long flags; +#if !MALI_USE_CSF ktime_t end_timestamp = ktime_get_raw(); +#endif struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; if (!kbdev->arb.arb_if) @@ -937,11 +991,14 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) /* Clear all jobs running on the GPU */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->protected_mode = false; +#if !MALI_USE_CSF kbase_backend_reset(kbdev, &end_timestamp); kbase_pm_metrics_update(kbdev, NULL); +#endif kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#if !MALI_USE_CSF /* Cancel any pending HWC dumps */ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING || @@ -951,6 +1008,7 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) wake_up(&kbdev->hwcnt.backend.wait); } spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); +#endif } mutex_unlock(&arb_vm_state->vm_state_lock); mutex_unlock(&kbdev->pm.lock); @@ -968,6 +1026,7 @@ int kbase_pm_force_mcu_wakeup_after_sleep(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); /* Set the override flag to force the power up of L2 cache */ kbdev->pm.backend.gpu_wakeup_override = true; + kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_NONE; kbase_pm_update_state(kbdev); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -1003,14 +1062,27 @@ static int pm_handle_mcu_sleep_on_runtime_suspend(struct kbase_device *kbdev) return ret; } - /* Check if a Doorbell mirror interrupt occurred meanwhile */ + /* Check if a Doorbell mirror interrupt occurred meanwhile. + * Also check if GPU idle work item is pending. If FW had sent the GPU idle notification + * after the wake up of MCU then it can be assumed that Userspace submission didn't make + * GPU non-idle, so runtime suspend doesn't need to be aborted. + */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (kbdev->pm.backend.gpu_sleep_mode_active && kbdev->pm.backend.exit_gpu_sleep_mode) { - dev_dbg(kbdev->dev, - "DB mirror interrupt occurred during runtime suspend after L2 power up"); - kbdev->pm.backend.gpu_wakeup_override = false; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return -EBUSY; + if (kbdev->pm.backend.gpu_sleep_mode_active && kbdev->pm.backend.exit_gpu_sleep_mode && + !work_pending(&kbdev->csf.scheduler.gpu_idle_work)) { + u32 glb_req = + kbase_csf_firmware_global_input_read(&kbdev->csf.global_iface, GLB_REQ); + u32 glb_ack = kbase_csf_firmware_global_output(&kbdev->csf.global_iface, GLB_ACK); + + /* Only abort the runtime suspend if GPU idle event is not pending */ + if (!((glb_req ^ glb_ack) & GLB_REQ_IDLE_EVENT_MASK)) { + dev_dbg(kbdev->dev, + "DB mirror interrupt occurred during runtime suspend after L2 power up"); + kbdev->pm.backend.gpu_wakeup_override = false; + kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_DB_MIRROR_IRQ; + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + return -EBUSY; + } } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* Need to release the kbdev->pm.lock to avoid lock ordering issue diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c index b16d8d99ad7e..8daef13388a3 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c @@ -109,13 +109,19 @@ unlock: KBASE_EXPORT_TEST_API(kbase_devfreq_set_core_mask); #endif -u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) +u64 kbase_pm_ca_get_debug_core_mask(struct kbase_device *kbdev) { #if MALI_USE_CSF - u64 debug_core_mask = kbdev->pm.debug_core_mask; + return kbdev->pm.debug_core_mask; #else - u64 debug_core_mask = kbdev->pm.debug_core_mask_all; + return kbdev->pm.debug_core_mask_all; #endif +} +KBASE_EXPORT_TEST_API(kbase_pm_ca_get_debug_core_mask); + +u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) +{ + u64 debug_core_mask = kbase_pm_ca_get_debug_core_mask(kbdev); lockdep_assert_held(&kbdev->hwaccess_lock); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h index 95ec1dfb0739..37d1020e28ff 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.h @@ -57,6 +57,17 @@ void kbase_pm_ca_term(struct kbase_device *kbdev); */ u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev); +/** + * kbase_pm_ca_get_debug_core_mask - Get debug core mask. + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + * + * Returns a mask of the currently selected shader cores. + * + * Return: The bit mask of user-selected cores + */ +u64 kbase_pm_ca_get_debug_core_mask(struct kbase_device *kbdev); + /** * kbase_pm_ca_update_core_status - Update core status * diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h index e5ae92a23c90..a0b8b9500077 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h @@ -29,6 +29,8 @@ #include "mali_kbase_pm_always_on.h" #include "mali_kbase_pm_coarse_demand.h" +#include + #if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM) #define KBASE_PM_RUNTIME 1 #endif @@ -97,6 +99,21 @@ enum kbase_shader_core_state { #undef KBASEP_SHADER_STATE }; +/** + * enum kbase_pm_runtime_suspend_abort_reason - Reason why runtime suspend was aborted + * after the wake up of MCU. + * + * @ABORT_REASON_NONE: Not aborted + * @ABORT_REASON_DB_MIRROR_IRQ: Runtime suspend was aborted due to DB_MIRROR irq. + * @ABORT_REASON_NON_IDLE_CGS: Runtime suspend was aborted as CSGs were detected as non-idle after + * their suspension. + */ +enum kbase_pm_runtime_suspend_abort_reason { + ABORT_REASON_NONE, + ABORT_REASON_DB_MIRROR_IRQ, + ABORT_REASON_NON_IDLE_CGS +}; + /** * struct kbasep_pm_metrics - Metrics data collected for use by the power * management framework. @@ -248,9 +265,6 @@ union kbase_pm_policy_data { * states and transitions. * @cg1_disabled: Set if the policy wants to keep the second core group * powered off - * @driver_ready_for_irqs: Debug state indicating whether sufficient - * initialization of the driver has occurred to handle - * IRQs * @metrics: Structure to hold metrics for the GPU * @shader_tick_timer: Structure to hold the shader poweroff tick timer state * @poweroff_wait_in_progress: true if a wait for GPU power off is in progress. @@ -290,6 +304,8 @@ union kbase_pm_policy_data { * called previously. * See &struct kbase_pm_callback_conf. * @ca_cores_enabled: Cores that are currently available + * @apply_hw_issue_TITANHW_2938_wa: Indicates if the workaround for BASE_HW_ISSUE_TITANHW_2938 + * needs to be applied when unmapping memory from GPU. * @mcu_state: The current state of the micro-control unit, only applicable * to GPUs that have such a component * @l2_state: The current state of the L2 cache state machine. See @@ -358,6 +374,12 @@ union kbase_pm_policy_data { * mode for the saving the HW state before power down. * @db_mirror_interrupt_enabled: Flag tracking if the Doorbell mirror interrupt * is enabled or not. + * @runtime_suspend_abort_reason: Tracks if the runtime suspend was aborted, + * after the wake up of MCU, due to the DB_MIRROR irq + * or non-idle CSGs. Tracking is done to avoid + * redundant transition of MCU to sleep state after the + * abort of runtime suspend and before the resumption + * of scheduling. * @l2_force_off_after_mcu_halt: Flag to indicate that L2 cache power down is * must after performing the MCU halt. Flag is set * immediately after the MCU halt and cleared @@ -427,10 +449,6 @@ struct kbase_pm_backend_data { bool cg1_disabled; -#ifdef CONFIG_MALI_BIFROST_DEBUG - bool driver_ready_for_irqs; -#endif /* CONFIG_MALI_BIFROST_DEBUG */ - struct kbasep_pm_metrics_state metrics; struct kbasep_pm_tick_timer_state shader_tick_timer; @@ -458,6 +476,7 @@ struct kbase_pm_backend_data { u64 ca_cores_enabled; #if MALI_USE_CSF + bool apply_hw_issue_TITANHW_2938_wa; enum kbase_mcu_state mcu_state; #endif enum kbase_l2_core_state l2_state; @@ -479,6 +498,7 @@ struct kbase_pm_backend_data { bool gpu_idled; bool gpu_wakeup_override; bool db_mirror_interrupt_enabled; + enum kbase_pm_runtime_suspend_abort_reason runtime_suspend_abort_reason; #endif bool l2_force_off_after_mcu_halt; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c index b77e46b35422..506e168f86d2 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c @@ -114,7 +114,8 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev) return true; #ifdef KBASE_PM_RUNTIME - if (kbdev->pm.backend.gpu_wakeup_override) + if (kbdev->pm.backend.gpu_wakeup_override || + kbdev->pm.backend.runtime_suspend_abort_reason != ABORT_REASON_NONE) return true; #endif @@ -351,12 +352,14 @@ __pure static u32 map_core_type_to_tl_pm_state(struct kbase_device *kbdev, } #endif -#if IS_ENABLED(CONFIG_ARM64) +#if IS_ENABLED(CONFIG_ARM64) && !MALI_USE_CSF + static void mali_cci_flush_l2(struct kbase_device *kbdev) { + u32 val; const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; - u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS; - u32 raw; + const u32 timeout_us = + kbase_get_timeout_ms(kbdev, KBASE_CLEAN_CACHE_TIMEOUT) * USEC_PER_MSEC; /* * Note that we don't take the cache flush mutex here since @@ -368,14 +371,11 @@ static void mali_cci_flush_l2(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), GPU_COMMAND_CACHE_CLN_INV_L2); - raw = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT)); - /* Wait for cache flush to complete before continuing, exit on * gpu resets or loop expiry. */ - while (((raw & mask) == 0) && --loops) { - raw = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT)); - } + kbase_reg_poll32_timeout(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT), val, val & mask, 0, + timeout_us, false); } #endif @@ -632,7 +632,7 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) val |= (kbdev->l2_hash_override << L2_CONFIG_HASH_SHIFT); } else if (kbdev->l2_hash_values_override) { #if MALI_USE_CSF - int i; + uint i; WARN_ON(!kbase_hw_has_l2_slice_hash_feature(kbdev)); @@ -640,7 +640,7 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) val |= (0x1 << L2_CONFIG_L2_SLICE_HASH_ENABLE_SHIFT); for (i = 0; i < GPU_L2_SLICE_HASH_COUNT; i++) { /* L2_SLICE_HASH and ASN_HASH alias each other */ - dev_dbg(kbdev->dev, "Program 0x%x to ASN_HASH[%d]\n", + dev_dbg(kbdev->dev, "Program 0x%x to ASN_HASH[%u]\n", kbdev->l2_hash_values[i], i); kbase_reg_write32(kbdev, GPU_L2_SLICE_HASH_OFFSET(i), kbdev->l2_hash_values[i]); @@ -757,19 +757,20 @@ static void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev) */ static void wait_mcu_as_inactive(struct kbase_device *kbdev) { - unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; - + u32 val; + int err; + const u32 timeout_us = + kbase_get_timeout_ms(kbdev, KBASE_AS_INACTIVE_TIMEOUT) * USEC_PER_MSEC; lockdep_assert_held(&kbdev->hwaccess_lock); if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_2716)) return; /* Wait for the AS_ACTIVE_INT bit to become 0 for the AS used by MCU FW */ - while (--max_loops && kbase_reg_read32(kbdev, MMU_AS_OFFSET(MCU_AS_NR, STATUS)) & - AS_STATUS_AS_ACTIVE_INT_MASK) - ; - - if (!WARN_ON_ONCE(max_loops == 0)) + err = kbase_reg_poll32_timeout(kbdev, MMU_AS_OFFSET(MCU_AS_NR, STATUS), val, + !(val & AS_STATUS_AS_ACTIVE_INT_MASK), 10, timeout_us, + false); + if (!WARN_ON_ONCE(err == -ETIMEDOUT)) return; dev_err(kbdev->dev, "AS_ACTIVE_INT bit stuck for AS %d used by MCU FW", MCU_AS_NR); @@ -1341,15 +1342,8 @@ static void kbase_pm_l2_clear_backend_slot_submit_kctx(struct kbase_device *kbde static bool can_power_down_l2(struct kbase_device *kbdev) { -#if MALI_USE_CSF - /* Due to the HW issue GPU2019-3878, need to prevent L2 power off - * whilst MMU command is in progress. - * Also defer the power-down if MMU is in process of page migration. - */ - return !kbdev->mmu_hw_operation_in_progress && !kbdev->mmu_page_migrate_in_progress; -#else + /* Defer the power-down if MMU is in process of page migration. */ return !kbdev->mmu_page_migrate_in_progress; -#endif } static bool can_power_up_l2(struct kbase_device *kbdev) @@ -1508,7 +1502,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) * must power them on explicitly. */ if (l2_present != 1) - kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present & ~1, + kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, l2_present & ~1ULL, ACTION_PWRON); /* Clear backend slot submission kctx */ kbase_pm_l2_clear_backend_slot_submit_kctx(kbdev); @@ -2501,7 +2495,8 @@ int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); #if MALI_USE_CSF - timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT)); + timeout = (unsigned long)kbase_csf_timeout_in_jiffies( + kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT)); #else timeout = msecs_to_jiffies(PM_TIMEOUT_MS); #endif @@ -2510,11 +2505,11 @@ int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev) #if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE remaining = wait_event_killable_timeout(kbdev->pm.backend.gpu_in_desired_state_wait, kbase_pm_is_in_desired_state_with_l2_powered(kbdev), - timeout); + (long)timeout); #else remaining = wait_event_timeout(kbdev->pm.backend.gpu_in_desired_state_wait, kbase_pm_is_in_desired_state_with_l2_powered(kbdev), - timeout); + (long)timeout); #endif if (!remaining) { @@ -2535,7 +2530,7 @@ static int pm_wait_for_desired_state(struct kbase_device *kbdev, bool killable_w #if MALI_USE_CSF long timeout = kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT)); #else - long timeout = msecs_to_jiffies(PM_TIMEOUT_MS); + long timeout = (long)msecs_to_jiffies(PM_TIMEOUT_MS); #endif int err = 0; @@ -2662,7 +2657,7 @@ static int pm_wait_for_poweroff_work_complete(struct kbase_device *kbdev, bool k /* Handling of timeout error isn't supported for arbiter builds */ const long timeout = MAX_SCHEDULE_TIMEOUT; #else - const long timeout = msecs_to_jiffies(PM_TIMEOUT_MS); + const long timeout = (long)msecs_to_jiffies(PM_TIMEOUT_MS); #endif #endif int err = 0; @@ -3081,7 +3076,7 @@ static int kbase_set_gpu_quirks(struct kbase_device *kbdev) #endif /* !MALI_USE_CSF */ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) { - int default_idvs_group_size = 0xF; + u32 default_idvs_group_size = 0xF; u32 group_size = 0; if (of_property_read_u32(kbdev->dev->of_node, "idvs-group-size", &group_size)) @@ -3219,6 +3214,7 @@ static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) { +#if !MALI_USE_CSF if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && !kbdev->cci_snoop_enabled) { #if IS_ENABLED(CONFIG_ARM64) if (kbdev->snoop_enable_smc != 0) @@ -3227,10 +3223,12 @@ void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n"); kbdev->cci_snoop_enabled = true; } +#endif /* !MALI_USE_CSF */ } void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) { +#if !MALI_USE_CSF if (kbdev->cci_snoop_enabled) { #if IS_ENABLED(CONFIG_ARM64) if (kbdev->snoop_disable_smc != 0) { @@ -3241,6 +3239,7 @@ void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n"); kbdev->cci_snoop_enabled = false; } +#endif /* !MALI_USE_CSF */ } #if !MALI_USE_CSF diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h index 851d56141e53..033c80a7c6b4 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h @@ -1041,4 +1041,5 @@ static inline bool kbase_pm_l2_allow_mmu_page_migration(struct kbase_device *kbd return (backend->l2_state != KBASE_L2_PEND_ON && backend->l2_state != KBASE_L2_PEND_OFF); } + #endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c index 5e6e9f058da9..833a8216928e 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c @@ -454,7 +454,7 @@ static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) { if (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { - int device_nr = + u32 device_nr = (katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) ? katom->device_nr : 0; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c index 59d38cad0031..23e447b15767 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c @@ -228,7 +228,8 @@ void kbase_pm_update_cores_state(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -int kbase_pm_list_policies(struct kbase_device *kbdev, const struct kbase_pm_policy *const **list) +size_t kbase_pm_list_policies(struct kbase_device *kbdev, + const struct kbase_pm_policy *const **list) { CSTD_UNUSED(kbdev); if (list) @@ -294,7 +295,7 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic bool reset_gpu = false; bool reset_op_prevented = true; struct kbase_csf_scheduler *scheduler = NULL; - u32 pwroff; + u64 pwroff_ns; bool switching_to_always_on; #endif @@ -304,9 +305,9 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic KBASE_KTRACE_ADD(kbdev, PM_SET_POLICY, NULL, new_policy->id); #if MALI_USE_CSF - pwroff = kbase_csf_firmware_get_mcu_core_pwroff_time(kbdev); + pwroff_ns = kbase_csf_firmware_get_mcu_core_pwroff_time(kbdev); switching_to_always_on = new_policy == &kbase_pm_always_on_policy_ops; - if (pwroff == 0 && !switching_to_always_on) { + if (pwroff_ns == 0 && !switching_to_always_on) { dev_warn( kbdev->dev, "power_policy: cannot switch away from always_on with mcu_shader_pwroff_timeout set to 0\n"); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c index f4ff61ff5eb6..0bf0f5a062d3 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c @@ -22,7 +22,6 @@ #include #include #if MALI_USE_CSF -#include #include #include #endif @@ -30,6 +29,11 @@ #include #include #include +#include + +#if !IS_ENABLED(CONFIG_MALI_REAL_HW) +#include +#endif struct kbase_timeout_info { char *selector_str; @@ -53,6 +57,15 @@ static struct kbase_timeout_info timeout_info[KBASE_TIMEOUT_SELECTOR_COUNT] = { MMU_AS_INACTIVE_WAIT_TIMEOUT_CYCLES }, [KCPU_FENCE_SIGNAL_TIMEOUT] = { "KCPU_FENCE_SIGNAL_TIMEOUT", KCPU_FENCE_SIGNAL_TIMEOUT_CYCLES }, + [KBASE_PRFCNT_ACTIVE_TIMEOUT] = { "KBASE_PRFCNT_ACTIVE_TIMEOUT", + KBASE_PRFCNT_ACTIVE_TIMEOUT_CYCLES }, + [KBASE_CLEAN_CACHE_TIMEOUT] = { "KBASE_CLEAN_CACHE_TIMEOUT", + KBASE_CLEAN_CACHE_TIMEOUT_CYCLES }, + [KBASE_AS_INACTIVE_TIMEOUT] = { "KBASE_AS_INACTIVE_TIMEOUT", + KBASE_AS_INACTIVE_TIMEOUT_CYCLES }, + [IPA_INACTIVE_TIMEOUT] = { "IPA_INACTIVE_TIMEOUT", IPA_INACTIVE_TIMEOUT_CYCLES }, + [CSF_FIRMWARE_STOP_TIMEOUT] = { "CSF_FIRMWARE_STOP_TIMEOUT", + CSF_FIRMWARE_STOP_TIMEOUT_CYCLES }, }; #else static struct kbase_timeout_info timeout_info[KBASE_TIMEOUT_SELECTOR_COUNT] = { @@ -60,6 +73,12 @@ static struct kbase_timeout_info timeout_info[KBASE_TIMEOUT_SELECTOR_COUNT] = { MMU_AS_INACTIVE_WAIT_TIMEOUT_CYCLES }, [JM_DEFAULT_JS_FREE_TIMEOUT] = { "JM_DEFAULT_JS_FREE_TIMEOUT", JM_DEFAULT_JS_FREE_TIMEOUT_CYCLES }, + [KBASE_PRFCNT_ACTIVE_TIMEOUT] = { "KBASE_PRFCNT_ACTIVE_TIMEOUT", + KBASE_PRFCNT_ACTIVE_TIMEOUT_CYCLES }, + [KBASE_CLEAN_CACHE_TIMEOUT] = { "KBASE_CLEAN_CACHE_TIMEOUT", + KBASE_CLEAN_CACHE_TIMEOUT_CYCLES }, + [KBASE_AS_INACTIVE_TIMEOUT] = { "KBASE_AS_INACTIVE_TIMEOUT", + KBASE_AS_INACTIVE_TIMEOUT_CYCLES }, }; #endif @@ -282,10 +301,23 @@ static void get_cpu_gpu_time(struct kbase_device *kbdev, u64 *cpu_ts, u64 *gpu_t kbase_backend_get_gpu_time(kbdev, gpu_cycle, gpu_ts, &ts); if (cpu_ts) - *cpu_ts = ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; + *cpu_ts = (u64)(ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec); } #endif +u64 kbase_arch_timer_get_cntfrq(struct kbase_device *kbdev) +{ + u64 freq = arch_timer_get_cntfrq(); + +#if !IS_ENABLED(CONFIG_MALI_REAL_HW) + freq = midgard_model_arch_timer_get_cntfrq(kbdev->model); +#endif + + dev_dbg(kbdev->dev, "System Timer Freq = %lluHz", freq); + + return freq; +} + int kbase_backend_time_init(struct kbase_device *kbdev) { int err = 0; @@ -297,7 +329,7 @@ int kbase_backend_time_init(struct kbase_device *kbdev) kbase_pm_register_access_enable(kbdev); get_cpu_gpu_time(kbdev, &cpu_ts, &gpu_ts, NULL); - freq = arch_timer_get_cntfrq(); + freq = kbase_arch_timer_get_cntfrq(kbdev); if (!freq) { dev_warn(kbdev->dev, "arch_timer_get_rate() is zero!"); @@ -316,8 +348,9 @@ int kbase_backend_time_init(struct kbase_device *kbdev) goto disable_registers; } - kbdev->backend_time.offset = cpu_ts - div64_u64(gpu_ts * kbdev->backend_time.multiplier, - kbdev->backend_time.divisor); + kbdev->backend_time.offset = + (s64)(cpu_ts - div64_u64(gpu_ts * kbdev->backend_time.multiplier, + kbdev->backend_time.divisor)); #endif if (kbase_timeout_scaling_init(kbdev)) { diff --git a/drivers/gpu/arm/bifrost/build.bp b/drivers/gpu/arm/bifrost/build.bp index 72dd15f2d8cc..9ee968af8de5 100644 --- a/drivers/gpu/arm/bifrost/build.bp +++ b/drivers/gpu/arm/bifrost/build.bp @@ -65,9 +65,6 @@ bob_defaults { large_page_support: { kbuild_options: ["CONFIG_LARGE_PAGE_SUPPORT=y"], }, - mali_memory_fully_backed: { - kbuild_options: ["CONFIG_MALI_MEMORY_FULLY_BACKED=y"], - }, mali_corestack: { kbuild_options: ["CONFIG_MALI_CORESTACK=y"], }, diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context.c b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c index 41f129624245..36cfde3cdab1 100644 --- a/drivers/gpu/arm/bifrost/context/mali_kbase_context.c +++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context.c @@ -188,7 +188,7 @@ int kbase_context_common_init(struct kbase_context *kctx) bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG); kbase_gpu_vm_unlock(kctx); - kctx->id = atomic_add_return(1, &(kctx->kbdev->ctx_num)) - 1; + kctx->id = (u32)atomic_add_return(1, &(kctx->kbdev->ctx_num)) - 1; mutex_lock(&kctx->kbdev->kctx_list_lock); err = kbase_insert_kctx_to_process(kctx); diff --git a/drivers/gpu/arm/bifrost/context/mali_kbase_context.h b/drivers/gpu/arm/bifrost/context/mali_kbase_context.h index 939eb9bbd65e..e2295d020292 100644 --- a/drivers/gpu/arm/bifrost/context/mali_kbase_context.h +++ b/drivers/gpu/arm/bifrost/context/mali_kbase_context.h @@ -88,7 +88,7 @@ void kbase_destroy_context(struct kbase_context *kctx); */ static inline bool kbase_ctx_flag(struct kbase_context *kctx, enum kbase_context_flags flag) { - return atomic_read(&kctx->flags) & flag; + return atomic_read(&kctx->flags) & (int)flag; } /** diff --git a/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c b/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c index 1489d8c1971b..61a4be9ccc94 100644 --- a/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c +++ b/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c @@ -45,11 +45,6 @@ */ #define TIMER_EVENTS_PER_SECOND ((u32)1000 / IPA_CONTROL_TIMER_DEFAULT_VALUE_MS) -/* - * Maximum number of loops polling the GPU before we assume the GPU has hung. - */ -#define IPA_INACTIVE_MAX_LOOPS (8000000U) - /* * Number of bits used to configure a performance counter in SELECT registers. */ @@ -86,16 +81,16 @@ static u32 timer_value(u32 gpu_rate) static int wait_status(struct kbase_device *kbdev, u32 flags) { - unsigned int max_loops = IPA_INACTIVE_MAX_LOOPS; - u32 status = kbase_reg_read32(kbdev, IPA_CONTROL_ENUM(STATUS)); - + u32 val; + const u32 timeout_us = kbase_get_timeout_ms(kbdev, IPA_INACTIVE_TIMEOUT) * USEC_PER_MSEC; /* * Wait for the STATUS register to indicate that flags have been * cleared, in case a transition is pending. */ - while (--max_loops && (status & flags)) - status = kbase_reg_read32(kbdev, IPA_CONTROL_ENUM(STATUS)); - if (max_loops == 0) { + const int err = kbase_reg_poll32_timeout(kbdev, IPA_CONTROL_ENUM(STATUS), val, + !(val & flags), 0, timeout_us, false); + + if (err) { dev_err(kbdev->dev, "IPA_CONTROL STATUS register stuck"); return -EBUSY; } @@ -126,7 +121,7 @@ static int apply_select_config(struct kbase_device *kbdev, u64 *select) return ret; } -static u64 read_value_cnt(struct kbase_device *kbdev, u8 type, int select_idx) +static u64 read_value_cnt(struct kbase_device *kbdev, u8 type, u8 select_idx) { switch (type) { case KBASE_IPA_CORE_TYPE_CSHW: diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c index 2cd4b201173d..9dffe34f095b 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c @@ -191,14 +191,14 @@ static int kernel_map_user_io_pages(struct kbase_context *kctx, struct kbase_que struct page *page_list[2]; pgprot_t cpu_map_prot; unsigned long flags; - uint64_t *user_io_addr; + u64 *user_io_addr; int ret = 0; size_t i; kbase_gpu_vm_lock(kctx); if (ARRAY_SIZE(page_list) > (KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES - - atomic_read(&kctx->permanent_mapped_pages))) { + (unsigned int)atomic_read(&kctx->permanent_mapped_pages))) { ret = -ENOMEM; goto unlock; } @@ -553,7 +553,7 @@ static int csf_queue_register_internal(struct kbase_context *kctx, * enabled, otherwise leave them as default zeros. */ if (reg_ex && reg_ex->ex_buffer_size) { - u32 cfg = CS_INSTR_CONFIG_EVENT_SIZE_SET(0, reg_ex->ex_event_size); + u32 cfg = CS_INSTR_CONFIG_EVENT_SIZE_SET(0U, reg_ex->ex_event_size); cfg = CS_INSTR_CONFIG_EVENT_STATE_SET(cfg, reg_ex->ex_event_state); queue->trace_cfg = cfg; @@ -724,7 +724,7 @@ int kbase_csf_queue_bind(struct kbase_context *kctx, union kbase_ioctl_cs_queue_ group->bound_queues[bind->in.csi_index] = queue; queue->group = group; queue->group_priority = group->priority; - queue->csi_index = bind->in.csi_index; + queue->csi_index = (s8)bind->in.csi_index; queue->bind_state = KBASE_CSF_QUEUE_BIND_IN_PROGRESS; out: @@ -840,8 +840,8 @@ void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, int csi_index dmb(osh); value = kbase_csf_firmware_csg_output(ginfo, CSG_DB_ACK); - value ^= (1 << csi_index); - kbase_csf_firmware_csg_input_mask(ginfo, CSG_DB_REQ, value, 1 << csi_index); + value ^= (1U << csi_index); + kbase_csf_firmware_csg_input_mask(ginfo, CSG_DB_REQ, value, 1U << csi_index); if (likely(ring_csg_doorbell)) kbase_csf_ring_csg_doorbell(kbdev, csg_nr); @@ -902,7 +902,7 @@ static void unbind_stopped_queue(struct kbase_context *kctx, struct kbase_queue unsigned long flags; kbase_csf_scheduler_spin_lock(kctx->kbdev, &flags); - bitmap_clear(queue->group->protm_pending_bitmap, queue->csi_index, 1); + bitmap_clear(queue->group->protm_pending_bitmap, (unsigned int)queue->csi_index, 1); KBASE_KTRACE_ADD_CSF_GRP_Q(kctx->kbdev, CSI_PROTM_PEND_CLEAR, queue->group, queue, queue->group->protm_pending_bitmap[0]); queue->group->bound_queues[queue->csi_index] = NULL; @@ -1933,8 +1933,8 @@ static void flush_gpu_cache_on_fatal_error(struct kbase_device *kbdev) */ if (kbdev->pm.backend.gpu_powered) { kbase_gpu_start_cache_clean(kbdev, GPU_COMMAND_CACHE_CLN_INV_L2_LSC); - if (kbase_gpu_wait_cache_clean_timeout(kbdev, - kbdev->mmu_or_gpu_cache_op_wait_time_ms)) + if (kbase_gpu_wait_cache_clean_timeout( + kbdev, kbase_get_timeout_ms(kbdev, MMU_AS_INACTIVE_WAIT_TIMEOUT))) dev_warn( kbdev->dev, "[%llu] Timeout waiting for CACHE_CLN_INV_L2_LSC to complete after fatal error", @@ -2268,12 +2268,13 @@ static void handle_fault_event(struct kbase_queue *const queue, const u32 cs_ack const u32 cs_fault_exception_data = CS_FAULT_EXCEPTION_DATA_GET(cs_fault); const u64 cs_fault_info_exception_data = CS_FAULT_INFO_EXCEPTION_DATA_GET(cs_fault_info); bool use_old_log_format = true; + bool skip_fault_report = kbase_ctx_flag(queue->kctx, KCTX_PAGE_FAULT_REPORT_SKIP); kbase_csf_scheduler_spin_lock_assert_held(kbdev); - if (use_old_log_format) + if (use_old_log_format && !skip_fault_report) dev_warn(kbdev->dev, "Ctx %d_%d Group %d CSG %d CSI: %d\n" "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n" @@ -2334,7 +2335,7 @@ static void report_queue_fatal_error(struct kbase_queue *const queue, u32 cs_fat return; error.payload.csg_error.handle = group->handle; - error.payload.csg_error.error.payload.fatal_queue.csi_index = queue->csi_index; + error.payload.csg_error.error.payload.fatal_queue.csi_index = (__u8)queue->csi_index; kbase_csf_event_add_error(queue->kctx, &group->error_fatal, &error); kbase_event_wakeup(queue->kctx); } @@ -2443,11 +2444,12 @@ static void handle_fatal_event(struct kbase_queue *const queue, const u32 cs_fatal_exception_data = CS_FATAL_EXCEPTION_DATA_GET(cs_fatal); const u64 cs_fatal_info_exception_data = CS_FATAL_INFO_EXCEPTION_DATA_GET(cs_fatal_info); bool use_old_log_format = true; + bool skip_fault_report = kbase_ctx_flag(queue->kctx, KCTX_PAGE_FAULT_REPORT_SKIP); kbase_csf_scheduler_spin_lock_assert_held(kbdev); - if (use_old_log_format) + if (use_old_log_format && !skip_fault_report) dev_warn(kbdev->dev, "Ctx %d_%d Group %d CSG %d CSI: %d\n" "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n" @@ -2508,17 +2510,17 @@ static void process_cs_interrupts(struct kbase_queue_group *const group, kbase_csf_scheduler_spin_lock_assert_held(kbdev); while (remaining != 0) { - int const i = ffs(remaining) - 1; + unsigned int const i = (unsigned int)ffs((int)remaining) - 1; struct kbase_queue *const queue = group->bound_queues[i]; - remaining &= ~(1 << i); + remaining &= ~(1U << i); /* The queue pointer can be NULL, but if it isn't NULL then it * cannot disappear since scheduler spinlock is held and before * freeing a bound queue it has to be first unbound which * requires scheduler spinlock. */ - if (queue && !WARN_ON(queue->csi_index != i)) { + if (queue && !WARN_ON(queue->csi_index != (s8)i)) { struct kbase_csf_cmd_stream_info const *const stream = &ginfo->streams[i]; u32 const cs_req = kbase_csf_firmware_cs_input_read(stream, CS_REQ); u32 const cs_ack = kbase_csf_firmware_cs_output(stream, CS_ACK); @@ -2622,7 +2624,7 @@ static void process_cs_interrupts(struct kbase_queue_group *const group, * * See process_cs_interrupts() for details of per-stream interrupt handling. */ -static void process_csg_interrupts(struct kbase_device *const kbdev, int const csg_nr, +static void process_csg_interrupts(struct kbase_device *const kbdev, u32 const csg_nr, struct irq_idle_and_protm_track *track) { struct kbase_csf_cmd_stream_group_info *ginfo; @@ -2631,7 +2633,7 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, int const c kbase_csf_scheduler_spin_lock_assert_held(kbdev); - if (WARN_ON((u32)csg_nr >= kbdev->csf.global_iface.group_num)) + if (WARN_ON(csg_nr >= kbdev->csf.global_iface.group_num)) return; ginfo = &kbdev->csf.global_iface.groups[csg_nr]; @@ -2665,10 +2667,10 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, int const c if (!group) return; - if (WARN_ON(kbase_csf_scheduler_group_get_slot_locked(group) != csg_nr)) + if (WARN_ON((u32)kbase_csf_scheduler_group_get_slot_locked(group) != csg_nr)) return; - KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_INTERRUPT_PROCESS_START, group, csg_nr); + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_INTERRUPT_PROCESS_START, group, (u64)csg_nr); kbase_csf_handle_csg_sync_update(kbdev, ginfo, group, req, ack); @@ -2683,7 +2685,7 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, int const c KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_IDLE_SET, group, scheduler->csg_slots_idle_mask[0]); KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_INTERRUPT_IDLE, group, req ^ ack); - dev_dbg(kbdev->dev, "Idle notification received for Group %u on slot %d\n", + dev_dbg(kbdev->dev, "Idle notification received for Group %u on slot %u\n", group->handle, csg_nr); if (atomic_read(&scheduler->non_idle_offslot_grps)) { @@ -2700,7 +2702,7 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, int const c if (group->scan_seq_num < track->idle_seq) { track->idle_seq = group->scan_seq_num; - track->idle_slot = csg_nr; + track->idle_slot = (s8)csg_nr; } } @@ -2712,7 +2714,7 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, int const c req ^ ack); dev_info( kbdev->dev, - "[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %d\n", + "[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u\n", kbase_backend_get_cycle_cnt(kbdev), group->handle, group->kctx->tgid, group->kctx->id, csg_nr); @@ -2974,10 +2976,10 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) kbase_csf_scheduler_spin_lock(kbdev, &flags); /* Looping through and track the highest idle and protm groups */ while (csg_interrupts != 0) { - int const csg_nr = ffs(csg_interrupts) - 1; + u32 const csg_nr = (u32)ffs((int)csg_interrupts) - 1; process_csg_interrupts(kbdev, csg_nr, &track); - csg_interrupts &= ~(1 << csg_nr); + csg_interrupts &= ~(1U << csg_nr); } /* Handle protm from the tracked information */ @@ -3048,10 +3050,13 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) if (deferred_handling_glb_idle_irq) { unsigned long flags; + bool invoke_pm_state_machine; kbase_csf_scheduler_spin_lock(kbdev, &flags); - kbase_csf_scheduler_process_gpu_idle_event(kbdev); + invoke_pm_state_machine = kbase_csf_scheduler_process_gpu_idle_event(kbdev); kbase_csf_scheduler_spin_unlock(kbdev, flags); + if (unlikely(invoke_pm_state_machine)) + kbase_pm_update_state(kbdev); } wake_up_all(&kbdev->csf.event_wait); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c index 736545c86c99..c885845bc62e 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c @@ -236,7 +236,6 @@ static ssize_t kbase_csf_debugfs_scheduler_state_set(struct file *file, const ch { struct kbase_device *kbdev = file->private_data; char buf[MAX_SCHED_STATE_STRING_LEN]; - ssize_t ret = count; CSTD_UNUSED(ppos); @@ -256,10 +255,10 @@ static ssize_t kbase_csf_debugfs_scheduler_state_set(struct file *file, const ch kbase_csf_scheduler_force_wakeup(kbdev); else { dev_dbg(kbdev->dev, "Bad scheduler state %s", buf); - ret = -EINVAL; + return -EINVAL; } - return ret; + return (ssize_t)count; } static const struct file_operations kbasep_csf_debugfs_scheduler_state_fops = { diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h index c90b531d36b7..8d7c896e1051 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h @@ -26,15 +26,16 @@ #ifndef _KBASE_CSF_DEFS_H_ #define _KBASE_CSF_DEFS_H_ -#include -#include - #include #include "mali_kbase_csf_firmware.h" -#include "mali_kbase_refcount_defs.h" #include "mali_kbase_csf_event.h" #include +#include + +#include +#include + #if IS_ENABLED(CONFIG_MALI_CORESIGHT) #include #endif /* IS_ENABLED(CONFIG_MALI_CORESIGHT) */ @@ -245,11 +246,11 @@ enum kbase_csf_scheduler_state { /** * enum kbase_queue_group_priority - Kbase internal relative priority list. * - * @KBASE_QUEUE_GROUP_PRIORITY_REALTIME: The realtime queue group priority. - * @KBASE_QUEUE_GROUP_PRIORITY_HIGH: The high queue group priority. - * @KBASE_QUEUE_GROUP_PRIORITY_MEDIUM: The medium queue group priority. - * @KBASE_QUEUE_GROUP_PRIORITY_LOW: The low queue group priority. - * @KBASE_QUEUE_GROUP_PRIORITY_COUNT: The number of priority levels. + * @KBASE_QUEUE_GROUP_PRIORITY_REALTIME: The realtime queue group priority. + * @KBASE_QUEUE_GROUP_PRIORITY_HIGH: The high queue group priority. + * @KBASE_QUEUE_GROUP_PRIORITY_MEDIUM: The medium queue group priority. + * @KBASE_QUEUE_GROUP_PRIORITY_LOW: The low queue group priority. + * @KBASE_QUEUE_GROUP_PRIORITY_COUNT: The number of priority levels. */ enum kbase_queue_group_priority { KBASE_QUEUE_GROUP_PRIORITY_REALTIME = 0, @@ -273,7 +274,12 @@ enum kbase_queue_group_priority { * @CSF_SCHED_PROTM_PROGRESS_TIMEOUT: Timeout used to prevent protected mode execution hang. * @MMU_AS_INACTIVE_WAIT_TIMEOUT: Maximum waiting time in ms for the completion * of a MMU operation. - * @KCPU_FENCE_SIGNAL_TIMEOUT: Waiting time in ms for triggering a KCPU queue sync state dump + * @KCPU_FENCE_SIGNAL_TIMEOUT: Waiting time in ms for triggering a KCPU queue sync state dump. + * @KBASE_PRFCNT_ACTIVE_TIMEOUT: Waiting time for prfcnt to be ready. + * @KBASE_CLEAN_CACHE_TIMEOUT: Waiting time for cache flush to complete. + * @KBASE_AS_INACTIVE_TIMEOUT: Waiting time for MCU address space to become inactive. + * @IPA_INACTIVE_TIMEOUT: Waiting time for IPA_CONTROL_STATUS flags to be cleared. + * @CSF_FIRMWARE_STOP_TIMEOUT: Waiting time for the firmware to stop. * @KBASE_TIMEOUT_SELECTOR_COUNT: Number of timeout selectors. Must be last in * the enum. * @KBASE_DEFAULT_TIMEOUT: Default timeout used when an invalid selector is passed @@ -289,6 +295,11 @@ enum kbase_timeout_selector { CSF_SCHED_PROTM_PROGRESS_TIMEOUT, MMU_AS_INACTIVE_WAIT_TIMEOUT, KCPU_FENCE_SIGNAL_TIMEOUT, + KBASE_PRFCNT_ACTIVE_TIMEOUT, + KBASE_CLEAN_CACHE_TIMEOUT, + KBASE_AS_INACTIVE_TIMEOUT, + IPA_INACTIVE_TIMEOUT, + CSF_FIRMWARE_STOP_TIMEOUT, /* Must be the last in the enum */ KBASE_TIMEOUT_SELECTOR_COUNT, @@ -505,9 +516,9 @@ struct kbase_protected_suspend_buffer { * @prepared_seq_num: Indicates the position of queue group in the list of * prepared groups to be scheduled. * @scan_seq_num: Scan out sequence number before adjusting for dynamic - * idle conditions. It is used for setting a group's - * onslot priority. It could differ from prepared_seq_number - * when there are idle groups. + * idle conditions. It represents a group's global priority + * for a running tick/tock. It could differ from + * prepared_seq_number when there are idle groups. * @faulted: Indicates that a GPU fault occurred for the queue group. * This flag persists until the fault has been queued to be * reported to userspace. @@ -538,6 +549,12 @@ struct kbase_protected_suspend_buffer { * It is accumulated on consecutive mapping attempt failures. On * reaching a preset limit, the group is regarded as suffered * a fatal error and triggers a fatal error notification. + * @sched_act_seq_num: Scheduling action sequence number to determine the CSG slot + * priority in tick/tock scheduling action. It could differ from + * scan_seq_num. The field is only meaningful if the CSG is on the + * Scheduler's schedulable list for a tick/tock, and used for + * determining the CSG slot priority when the group is to be placed + * on the CSG slot. */ struct kbase_queue_group { struct kbase_context *kctx; @@ -589,6 +606,7 @@ struct kbase_queue_group { #endif void *csg_reg; u8 csg_reg_bind_retries; + u32 sched_act_seq_num; #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) /** * @prev_act: Previous CSG activity transition in a GPU metrics. @@ -729,13 +747,15 @@ struct kbase_csf_ctx_heap_reclaim_info { * GPU command queues are idle and at least one of them * is blocked on a sync wait operation. * @num_idle_wait_grps: Length of the @idle_wait_groups list. - * @sync_update_wq: Dedicated workqueue to process work items corresponding - * to the sync_update events by sync_set/sync_add - * instruction execution on CSs bound to groups - * of @idle_wait_groups list. - * @sync_update_work: work item to process the sync_update events by - * sync_set / sync_add instruction execution on command - * streams bound to groups of @idle_wait_groups list. + * @sync_update_wq_high_prio: high-priority work queue to process the + * SYNC_UPDATE events by sync_set / sync_add + * instruction execution on command streams bound to + * groups of @idle_wait_groups list. This WQ would + * be used if the context is prioritised. + * @sync_update_wq_normal_prio: similar to sync_update_wq_high_prio, but this + * WQ would be used if the context is not + * prioritised. + * @sync_update_work: Work item to process the SYNC_UPDATE events. * @ngrp_to_schedule: Number of groups added for the context to the * 'groups_to_schedule' list of scheduler instance. * @heap_info: Heap reclaim information data of the kctx. As the @@ -748,7 +768,8 @@ struct kbase_csf_scheduler_context { u32 num_runnable_grps; struct list_head idle_wait_groups; u32 num_idle_wait_grps; - struct workqueue_struct *sync_update_wq; + struct workqueue_struct *sync_update_wq_high_prio; + struct workqueue_struct *sync_update_wq_normal_prio; struct work_struct sync_update_work; u32 ngrp_to_schedule; struct kbase_csf_ctx_heap_reclaim_info heap_info; @@ -844,6 +865,11 @@ struct kbase_csf_user_reg_context { * @wq: Dedicated workqueue to process work items corresponding * to the OoM events raised for chunked tiler heaps being * used by GPU command queues, and progress timeout events. + * @kcpu_wq_high_prio: High-priority work queue to process KCPU commands for + * all queues in this context. This WQ would be used if + * the context is prioritised. + * @kcpu_wq_normal_prio: Similar to kcpu_wq_high_prio, but this WQ would be + * used if the context is not prioritised. * @link: Link to this csf context in the 'runnable_kctxs' list of * the scheduler instance * @sched: Object representing the scheduler's context @@ -862,6 +888,8 @@ struct kbase_csf_context { struct kbase_csf_event event; struct kbase_csf_tiler_heap_context tiler_heaps; struct workqueue_struct *wq; + struct workqueue_struct *kcpu_wq_high_prio; + struct workqueue_struct *kcpu_wq_normal_prio; struct list_head link; struct kbase_csf_scheduler_context sched; struct kbase_csf_cpu_queue_context cpu_queue; @@ -1069,6 +1097,9 @@ struct kbase_csf_mcu_shared_regions { * protected mode execution compared to other such * groups. It is updated on every tick/tock. * @interrupt_lock is used to serialize the access. + * @csg_scan_sched_count: Scheduling action counter used to assign the sched_act_seq_num + * for each group added to Scheduler's schedulable list in a + * tick/tock. * @protm_enter_time: GPU protected mode enter time. * @reclaim_mgr: CSGs tiler heap manager object. * @mcu_regs_data: Scheduler MCU shared regions data for managing the @@ -1118,6 +1149,7 @@ struct kbase_csf_scheduler { u32 pm_active_count; unsigned int csg_scheduling_period_ms; u32 tick_protm_pending_seq; + u32 csg_scan_sched_count; ktime_t protm_enter_time; struct kbase_csf_sched_heap_reclaim_mgr reclaim_mgr; struct kbase_csf_mcu_shared_regions mcu_regs_data; @@ -1201,6 +1233,8 @@ enum kbase_ipa_core_type { KBASE_IPA_CORE_TYPE_MEMSYS, KBASE_IPA_CORE_TYPE_TILER, KBASE_IPA_CORE_TYPE_SHADER, + + /* Must be the last in the enum */ KBASE_IPA_CORE_TYPE_NUM }; @@ -1328,7 +1362,7 @@ struct kbase_ipa_control { * @num_pages: Number of entries in @phys and @pma (and length of the interface) * @num_pages_aligned: Same as @num_pages except for the case when @is_small_page * is false and @reuse_pages is false and therefore will be - * aligned to NUM_4K_PAGES_IN_2MB_PAGE. + * aligned to NUM_PAGES_IN_2MB_LARGE_PAGE. * @virtual: Starting GPU virtual address this interface is mapped at * @flags: bitmask of CSF_FIRMWARE_ENTRY_* conveying the interface attributes * @data_start: Offset into firmware image at which the interface data starts @@ -1593,9 +1627,9 @@ struct kbase_csf_user_reg { * @mcu_core_pwroff_dur_count: The counterpart of the glb_pwroff timeout input * in interface required format, ready to be used * directly in the firmware. - * @mcu_core_pwroff_dur_count_modifier: Update csffw_glb_req_cfg_pwroff_timer - * to make the shr(10) modifier conditional - * on new flag in GLB_PWROFF_TIMER_CONFIG + * @mcu_core_pwroff_dur_count_no_modifier: Update csffw_glb_req_cfg_pwroff_timer + * to make the shr(10) modifier conditional + * on new flag in GLB_PWROFF_TIMER_CONFIG * @mcu_core_pwroff_reg_shadow: The actual value that has been programed into * the glb_pwoff register. This is separated from * the @p mcu_core_pwroff_dur_count as an update @@ -1606,10 +1640,9 @@ struct kbase_csf_user_reg { * @gpu_idle_dur_count: The counterpart of the hysteresis time window in * interface required format, ready to be used * directly in the firmware. - * @gpu_idle_dur_count_modifier: Update csffw_glb_req_idle_enable to make the shr(10) - * modifier conditional on the new flag - * in GLB_IDLE_TIMER_CONFIG. - * @fw_timeout_ms: Timeout value (in milliseconds) used when waiting + * @gpu_idle_dur_count_no_modifier: Update csffw_glb_req_idle_enable to make the shr(10) + * modifier conditional on the new flag + * in GLB_IDLE_TIMER_CONFIG. * for any request sent to the firmware. * @hwcnt: Contain members required for handling the dump of * HW counters. @@ -1656,14 +1689,13 @@ struct kbase_csf_device { bool glb_init_request_pending; struct work_struct fw_error_work; struct kbase_ipa_control ipa_control; - u32 mcu_core_pwroff_dur_ns; + u64 mcu_core_pwroff_dur_ns; u32 mcu_core_pwroff_dur_count; - u32 mcu_core_pwroff_dur_count_modifier; + u32 mcu_core_pwroff_dur_count_no_modifier; u32 mcu_core_pwroff_reg_shadow; - u32 gpu_idle_hysteresis_ns; + u64 gpu_idle_hysteresis_ns; u32 gpu_idle_dur_count; - u32 gpu_idle_dur_count_modifier; - unsigned int fw_timeout_ms; + u32 gpu_idle_dur_count_no_modifier; struct kbase_csf_hwcnt hwcnt; struct kbase_csf_mcu_fw fw; struct kbase_csf_firmware_log fw_log; @@ -1699,7 +1731,7 @@ struct kbase_csf_device { * @current_setup: Stores the MMU configuration for this address space. */ struct kbase_as { - int number; + unsigned int number; struct workqueue_struct *pf_wq; struct work_struct work_pagefault; struct work_struct work_busfault; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c index f2362d58cfa8..8ad1d74b64d5 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c @@ -66,6 +66,7 @@ static unsigned int csf_firmware_boot_timeout_ms; module_param(csf_firmware_boot_timeout_ms, uint, 0444); MODULE_PARM_DESC(csf_firmware_boot_timeout_ms, "Maximum time to wait for firmware to boot."); + #ifdef CONFIG_MALI_BIFROST_DEBUG /* Makes Driver wait indefinitely for an acknowledgment for the different * requests it sends to firmware. Otherwise the timeouts interfere with the @@ -110,8 +111,6 @@ MODULE_PARM_DESC(fw_debug, "Enables effective use of a debugger for debugging fi #define BUILD_INFO_GIT_DIRTY_LEN (1U) #define BUILD_INFO_GIT_SHA_PATTERN "git_sha: " -#define CSF_MAX_FW_STOP_LOOPS (100000) - #define CSF_GLB_REQ_CFG_MASK \ (GLB_REQ_CFG_ALLOC_EN_MASK | GLB_REQ_CFG_PROGRESS_TIMER_MASK | \ GLB_REQ_CFG_PWROFF_TIMER_MASK | GLB_REQ_IDLE_ENABLE_MASK) @@ -208,17 +207,6 @@ static int setup_shared_iface_static_region(struct kbase_device *kbdev) return ret; } -static int wait_mcu_status_value(struct kbase_device *kbdev, u32 val) -{ - u32 max_loops = CSF_MAX_FW_STOP_LOOPS; - - /* wait for the MCU_STATUS register to reach the given status value */ - while (--max_loops && (kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_STATUS)) != val)) - ; - - return (max_loops == 0) ? -1 : 0; -} - void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev) { @@ -229,10 +217,15 @@ void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev) static void wait_for_firmware_stop(struct kbase_device *kbdev) { - if (wait_mcu_status_value(kbdev, MCU_CONTROL_REQ_DISABLE) < 0) { - /* This error shall go away once MIDJM-2371 is closed */ - dev_err(kbdev->dev, "Firmware failed to stop"); - } + u32 val; + const u32 timeout_us = + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_STOP_TIMEOUT) * USEC_PER_MSEC; + const int err = kbase_reg_poll32_timeout(kbdev, GPU_CONTROL_ENUM(MCU_STATUS), val, + val == MCU_CONTROL_REQ_DISABLE, 0, timeout_us, + false); + + if (err) + dev_err(kbdev->dev, "Firmware failed to stop, error no: %d", err); KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF(kbdev, kbase_backend_get_cycle_cnt(kbdev)); } @@ -302,22 +295,15 @@ static void boot_csf_firmware(struct kbase_device *kbdev) */ static int wait_ready(struct kbase_device *kbdev) { - const ktime_t wait_loop_start = ktime_get_raw(); - const u32 mmu_as_inactive_wait_time_ms = kbdev->mmu_or_gpu_cache_op_wait_time_ms; - s64 diff; + u32 val; + const u32 timeout_us = + kbase_get_timeout_ms(kbdev, MMU_AS_INACTIVE_WAIT_TIMEOUT) * USEC_PER_MSEC; + const int err = kbase_reg_poll32_timeout(kbdev, MMU_AS_OFFSET(MCU_AS_NR, STATUS), val, + !(val & AS_STATUS_AS_ACTIVE_EXT_MASK), 0, + timeout_us, false); - do { - unsigned int i; - - for (i = 0; i < 1000; i++) { - /* Wait for the MMU status to indicate there is no active command */ - if (!(kbase_reg_read32(kbdev, MMU_AS_OFFSET(MCU_AS_NR, STATUS)) & - AS_STATUS_AS_ACTIVE_EXT_MASK)) - return 0; - } - - diff = ktime_to_ms(ktime_sub(ktime_get_raw(), wait_loop_start)); - } while (diff < mmu_as_inactive_wait_time_ms); + if (!err) + return 0; dev_err(kbdev->dev, "AS_ACTIVE bit stuck for MCU AS. Might be caused by unstable GPU clk/pwr or faulty system"); @@ -520,11 +506,11 @@ out: * parsed FW interface entry using large page(s) from protected memory. * If no appropriate entry is found it is set to NULL. * @num_pages: Number of pages requested. - * @num_pages_aligned: This is an output parameter used to carry the number of 4KB pages + * @num_pages_aligned: This is an output parameter used to carry the number of small pages * within the 2MB pages aligned allocation. * @is_small_page: This is an output flag used to select between the small and large page * to be used for the FW entry allocation. - * @force_small_page: Use 4kB pages to allocate memory needed for FW loading + * @force_small_page: Use small pages to allocate memory needed for FW loading * * Go through all the already initialized interfaces and find if a previously * allocated large page can be used to store contents of new FW interface entry. @@ -559,16 +545,16 @@ static inline bool entry_find_large_page_to_reuse(struct kbase_device *kbdev, * then use 2MB page(s) for it. */ if (!(virtual_start & (SZ_2M - 1))) { - *num_pages_aligned = round_up(*num_pages_aligned, NUM_4K_PAGES_IN_2MB_PAGE); + *num_pages_aligned = round_up(*num_pages_aligned, NUM_PAGES_IN_2MB_LARGE_PAGE); *is_small_page = false; goto out; } /* If the section doesn't lie within the same 2MB aligned boundary, - * then use 4KB pages as it would be complicated to use a 2MB page + * then use small pages as it would be complicated to use a 2MB page * for such section. */ - if ((virtual_start & ~(SZ_2M - 1)) != (virtual_end & ~(SZ_2M - 1))) + if ((virtual_start & ~(SZ_2M - 1UL)) != (virtual_end & ~(SZ_2M - 1UL))) goto out; /* Find the nearest 2MB aligned section which comes before the current @@ -599,7 +585,7 @@ static inline bool entry_find_large_page_to_reuse(struct kbase_device *kbdev, *phys = &target_interface->phys[page_index]; if (target_interface->pma) - *pma = &target_interface->pma[page_index / NUM_4K_PAGES_IN_2MB_PAGE]; + *pma = &target_interface->pma[page_index / NUM_PAGES_IN_2MB_LARGE_PAGE]; *is_small_page = false; reuse_large_page = true; @@ -1520,7 +1506,7 @@ static void handle_internal_firmware_fatal(struct kbase_device *const kbdev) * active address space and retain its refcount. */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as); + kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, (size_t)as); if (kctx) { kbase_ctx_sched_retain_ctx_refcount(kctx); @@ -1597,7 +1583,8 @@ static int wait_for_global_request_with_timeout(struct kbase_device *const kbdev static int wait_for_global_request(struct kbase_device *const kbdev, u32 const req_mask) { - return wait_for_global_request_with_timeout(kbdev, req_mask, kbdev->csf.fw_timeout_ms); + return wait_for_global_request_with_timeout( + kbdev, req_mask, kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); } static void set_global_request(const struct kbase_csf_global_iface *const global_iface, @@ -1635,7 +1622,7 @@ static void set_shader_poweroff_timer(struct kbase_device *const kbdev, kbase_csf_firmware_global_input(global_iface, GLB_PWROFF_TIMER, pwroff_reg); kbase_csf_firmware_global_input_mask(global_iface, GLB_PWROFF_TIMER_CONFIG, - kbdev->csf.mcu_core_pwroff_dur_count_modifier, + kbdev->csf.mcu_core_pwroff_dur_count_no_modifier, GLB_PWROFF_TIMER_CONFIG_NO_MODIFIER_MASK); set_global_request(global_iface, GLB_REQ_CFG_PWROFF_TIMER_MASK); @@ -1665,7 +1652,7 @@ static void enable_gpu_idle_timer(struct kbase_device *const kbdev) kbdev->csf.gpu_idle_dur_count); kbase_csf_firmware_global_input_mask(global_iface, GLB_IDLE_TIMER_CONFIG, - kbdev->csf.gpu_idle_dur_count_modifier, + kbdev->csf.gpu_idle_dur_count_no_modifier, GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK); kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_ENABLE, @@ -1961,11 +1948,11 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) kbase_pm_update_state(kbdev); } -static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_ns, u32 *modifier) +static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u64 dur_ns, u32 *no_modifier) { #define HYSTERESIS_VAL_UNIT_SHIFT (10) /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ - u64 freq = arch_timer_get_cntfrq(); + u64 freq = kbase_arch_timer_get_cntfrq(kbdev); u64 dur_val = dur_ns; u32 cnt_val_u32, reg_val_u32; bool src_system_timestamp = freq > 0; @@ -1988,10 +1975,10 @@ static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_n dur_val = dur_val * freq; dur_val = div_u64(dur_val, NSEC_PER_SEC); if (dur_val < S32_MAX) { - *modifier = 1; + *no_modifier = 1; } else { dur_val = dur_val >> HYSTERESIS_VAL_UNIT_SHIFT; - *modifier = 0; + *no_modifier = 0; } /* Interface limits the value field to S32_MAX */ @@ -2006,24 +1993,24 @@ static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_n return reg_val_u32; } -u32 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev) +u64 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev) { unsigned long flags; - u32 dur; + u64 dur_ns; kbase_csf_scheduler_spin_lock(kbdev, &flags); - dur = kbdev->csf.gpu_idle_hysteresis_ns; + dur_ns = kbdev->csf.gpu_idle_hysteresis_ns; kbase_csf_scheduler_spin_unlock(kbdev, flags); - return dur; + return dur_ns; } -u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u32 dur_ns) +u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u64 dur_ns) { unsigned long flags; - u32 modifier = 0; + u32 no_modifier = 0; - const u32 hysteresis_val = convert_dur_to_idle_count(kbdev, dur_ns, &modifier); + const u32 hysteresis_val = convert_dur_to_idle_count(kbdev, dur_ns, &no_modifier); /* The 'fw_load_lock' is taken to synchronize against the deferred * loading of FW, where the idle timer will be enabled. @@ -2033,7 +2020,7 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, kbase_csf_scheduler_spin_lock(kbdev, &flags); kbdev->csf.gpu_idle_hysteresis_ns = dur_ns; kbdev->csf.gpu_idle_dur_count = hysteresis_val; - kbdev->csf.gpu_idle_dur_count_modifier = modifier; + kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; kbase_csf_scheduler_spin_unlock(kbdev, flags); mutex_unlock(&kbdev->fw_load_lock); goto end; @@ -2073,7 +2060,7 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, kbase_csf_scheduler_spin_lock(kbdev, &flags); kbdev->csf.gpu_idle_hysteresis_ns = dur_ns; kbdev->csf.gpu_idle_dur_count = hysteresis_val; - kbdev->csf.gpu_idle_dur_count_modifier = modifier; + kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; kbase_csf_firmware_enable_gpu_idle_timer(kbdev); kbase_csf_scheduler_spin_unlock(kbdev, flags); wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK); @@ -2086,12 +2073,13 @@ end: return hysteresis_val; } +KBASE_EXPORT_TEST_API(kbase_csf_firmware_set_gpu_idle_hysteresis_time); -static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u32 dur_ns, - u32 *modifier) +static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u64 dur_ns, + u32 *no_modifier) { /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ - u64 freq = arch_timer_get_cntfrq(); + u64 freq = kbase_arch_timer_get_cntfrq(kbdev); u64 dur_val = dur_ns; u32 cnt_val_u32, reg_val_u32; bool src_system_timestamp = freq > 0; @@ -2117,10 +2105,10 @@ static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u3 dur_val = dur_val * freq; dur_val = div_u64(dur_val, NSEC_PER_SEC); if (dur_val < S32_MAX) { - *modifier = 1; + *no_modifier = 1; } else { dur_val = dur_val >> HYSTERESIS_VAL_UNIT_SHIFT; - *modifier = 0; + *no_modifier = 0; } if (dur_val == 0 && !always_on) { @@ -2143,29 +2131,29 @@ static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u3 return reg_val_u32; } -u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev) +u64 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev) { - u32 pwroff; + u64 pwroff_ns; unsigned long flags; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - pwroff = kbdev->csf.mcu_core_pwroff_dur_ns; + pwroff_ns = kbdev->csf.mcu_core_pwroff_dur_ns; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return pwroff; + return pwroff_ns; } -u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32 dur_ns) +u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u64 dur_ns) { unsigned long flags; - u32 modifier = 0; + u32 no_modifier = 0; - const u32 pwroff = convert_dur_to_core_pwroff_count(kbdev, dur_ns, &modifier); + const u32 pwroff = convert_dur_to_core_pwroff_count(kbdev, dur_ns, &no_modifier); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->csf.mcu_core_pwroff_dur_ns = dur_ns; kbdev->csf.mcu_core_pwroff_dur_count = pwroff; - kbdev->csf.mcu_core_pwroff_dur_count_modifier = modifier; + kbdev->csf.mcu_core_pwroff_dur_count_no_modifier = no_modifier; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); dev_dbg(kbdev->dev, "MCU shader Core Poweroff input update: 0x%.8x", pwroff); @@ -2178,6 +2166,34 @@ u32 kbase_csf_firmware_reset_mcu_core_pwroff_time(struct kbase_device *kbdev) return kbase_csf_firmware_set_mcu_core_pwroff_time(kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_NS); } +/** + * kbase_csf_get_iterator_trace_enable - Parsing the iterator_trace enable firstly from + * the module parameter, and then from device-tree. + * @kbdev: Kernel base device pointer + * + * Return: true on enabled, otherwise false. + */ +static bool kbase_csf_get_iterator_trace_enable(struct kbase_device *kbdev) +{ + const void *dt_iter_trace_param; + unsigned int val; + + + /* check device tree for iterator trace enable property and + * fallback to "iter_trace_enable" if not found and try again + */ + dt_iter_trace_param = of_get_property(kbdev->dev->of_node, "iter-trace-enable", NULL); + + if (!dt_iter_trace_param) + dt_iter_trace_param = + of_get_property(kbdev->dev->of_node, "iter_trace_enable", NULL); + + val = (dt_iter_trace_param) ? be32_to_cpup(dt_iter_trace_param) : 0; + dev_dbg(kbdev->dev, "Iterator trace enable device-tree config value: %u", val); + + return (val != 0); +} + /** * kbase_device_csf_iterator_trace_init - Send request to enable iterator * trace port. @@ -2188,50 +2204,35 @@ u32 kbase_csf_firmware_reset_mcu_core_pwroff_time(struct kbase_device *kbdev) */ static int kbase_device_csf_iterator_trace_init(struct kbase_device *kbdev) { - /* Enable the iterator trace port if supported by the GPU. - * It requires the GPU to have a nonzero "iter-trace-enable" - * property in the device tree, and the FW must advertise - * this feature in GLB_FEATURES. + /* Enable the iterator trace port if supported by the GPU and is + * configured to do so. The FW must advertise this feature in GLB_FEATURES. */ if (kbdev->pm.backend.gpu_powered) { - /* check device tree for iterator trace enable property - * and fallback to "iter_trace_enable" if it is not found - */ - const void *iter_trace_param = - of_get_property(kbdev->dev->of_node, "iter-trace-enable", NULL); - const struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; + bool dev_support_iter_trace = iface->features & + GLB_FEATURES_ITER_TRACE_SUPPORTED_MASK; - if (!iter_trace_param) - iter_trace_param = - of_get_property(kbdev->dev->of_node, "iter_trace_enable", NULL); + dev_dbg(kbdev->dev, "Device supporting iterator trace: %s\n", + dev_support_iter_trace ? "true" : "false"); + if (dev_support_iter_trace && kbase_csf_get_iterator_trace_enable(kbdev)) { + long ack_timeout = kbase_csf_timeout_in_jiffies( + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); - if (iter_trace_param) { - u32 iter_trace_value = be32_to_cpup(iter_trace_param); + /* write enable request to global input */ + kbase_csf_firmware_global_input_mask(iface, GLB_REQ, + GLB_REQ_ITER_TRACE_ENABLE_MASK, + GLB_REQ_ITER_TRACE_ENABLE_MASK); + /* Ring global doorbell */ + kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); - if ((iface->features & GLB_FEATURES_ITER_TRACE_SUPPORTED_MASK) && - iter_trace_value) { - long ack_timeout; + ack_timeout = wait_event_timeout( + kbdev->csf.event_wait, + !((kbase_csf_firmware_global_input_read(iface, GLB_REQ) ^ + kbase_csf_firmware_global_output(iface, GLB_ACK)) & + GLB_REQ_ITER_TRACE_ENABLE_MASK), + ack_timeout); - ack_timeout = kbase_csf_timeout_in_jiffies( - kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); - - /* write enable request to global input */ - kbase_csf_firmware_global_input_mask( - iface, GLB_REQ, GLB_REQ_ITER_TRACE_ENABLE_MASK, - GLB_REQ_ITER_TRACE_ENABLE_MASK); - /* Ring global doorbell */ - kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); - - ack_timeout = wait_event_timeout( - kbdev->csf.event_wait, - !((kbase_csf_firmware_global_input_read(iface, GLB_REQ) ^ - kbase_csf_firmware_global_output(iface, GLB_ACK)) & - GLB_REQ_ITER_TRACE_ENABLE_MASK), - ack_timeout); - - return ack_timeout ? 0 : -EINVAL; - } + return ack_timeout ? 0 : -EINVAL; } } return 0; @@ -2241,8 +2242,6 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev) { init_waitqueue_head(&kbdev->csf.event_wait); - kbdev->csf.fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); - kbase_csf_firmware_reset_mcu_core_pwroff_time(kbdev); INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces); INIT_LIST_HEAD(&kbdev->csf.firmware_config); @@ -2270,7 +2269,7 @@ void kbase_csf_firmware_early_term(struct kbase_device *kbdev) int kbase_csf_firmware_late_init(struct kbase_device *kbdev) { - u32 modifier = 0; + u32 no_modifier = 0; kbdev->csf.gpu_idle_hysteresis_ns = FIRMWARE_IDLE_HYSTERESIS_TIME_NS; @@ -2280,8 +2279,8 @@ int kbase_csf_firmware_late_init(struct kbase_device *kbdev) #endif WARN_ON(!kbdev->csf.gpu_idle_hysteresis_ns); kbdev->csf.gpu_idle_dur_count = - convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &modifier); - kbdev->csf.gpu_idle_dur_count_modifier = modifier; + convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &no_modifier); + kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; return 0; } @@ -2694,7 +2693,9 @@ int kbase_csf_firmware_mcu_register_read(struct kbase_device *const kbdev, u32 c int kbase_csf_firmware_mcu_register_poll(struct kbase_device *const kbdev, u32 const reg_addr, u32 const val_mask, u32 const reg_val) { - unsigned long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms) + jiffies; + unsigned long remaining = + kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)) + + jiffies; u32 read_val; dev_dbg(kbdev->dev, "p: reg %08x val %08x mask %08x", reg_addr, reg_val, val_mask); @@ -2922,6 +2923,12 @@ int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev) unsigned long flags; int err = 0; + err = kbase_reset_gpu_prevent_and_wait(kbdev); + if (err) { + dev_warn(kbdev->dev, "Unsuccessful GPU reset detected when updating FW config"); + return err; + } + /* Ensure GPU is powered-up until we complete config update.*/ kbase_csf_scheduler_pm_active(kbdev); err = kbase_csf_scheduler_killable_wait_mcu_active(kbdev); @@ -2944,6 +2951,7 @@ int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev) exit: kbase_csf_scheduler_pm_idle(kbdev); + kbase_reset_gpu_allow(kbdev); return err; } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h index b6f07d001d24..a2948a98e9a7 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h @@ -24,6 +24,7 @@ #include "device/mali_kbase_device.h" #include +#include #include /* @@ -770,9 +771,9 @@ extern bool fw_debug; static inline long kbase_csf_timeout_in_jiffies(const unsigned int msecs) { #ifdef CONFIG_MALI_BIFROST_DEBUG - return (fw_debug ? MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies(msecs)); + return (fw_debug ? MAX_SCHEDULE_TIMEOUT : (long)msecs_to_jiffies(msecs)); #else - return msecs_to_jiffies(msecs); + return (long)msecs_to_jiffies(msecs); #endif } @@ -807,15 +808,15 @@ void kbase_csf_firmware_disable_gpu_idle_timer(struct kbase_device *kbdev); * * Return: the internally recorded hysteresis (nominal) value. */ -u32 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev); +u64 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev); /** * kbase_csf_firmware_set_gpu_idle_hysteresis_time - Set the firmware GPU idle * detection hysteresis duration * - * @kbdev: Instance of a GPU platform device that implements a CSF interface. - * @dur: The duration value (unit: milliseconds) for the configuring - * hysteresis field for GPU idle detection + * @kbdev: Instance of a GPU platform device that implements a CSF interface. + * @dur_ns: The duration value (unit: nanoseconds) for the configuring + * hysteresis field for GPU idle detection * * The supplied value will be recorded internally without any change. But the * actual field value will be subject to hysteresis source frequency scaling @@ -827,7 +828,7 @@ u32 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev); * * Return: the actual internally configured hysteresis field value. */ -u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u32 dur); +u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u64 dur_ns); /** * kbase_csf_firmware_get_mcu_core_pwroff_time - Get the MCU shader Core power-off @@ -838,14 +839,14 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, * Return: the internally recorded MCU shader Core power-off (nominal) timeout value. The unit * of the value is in micro-seconds. */ -u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev); +u64 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev); /** * kbase_csf_firmware_set_mcu_core_pwroff_time - Set the MCU shader Core power-off * time value * * @kbdev: Instance of a GPU platform device that implements a CSF interface. - * @dur: The duration value (unit: micro-seconds) for configuring MCU + * @dur_ns: The duration value (unit: nanoseconds) for configuring MCU * core power-off timer, when the shader cores' power * transitions are delegated to the MCU (normal operational * mode) @@ -864,7 +865,7 @@ u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev); * Return: the actual internal core power-off timer value in register defined * format. */ -u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32 dur); +u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u64 dur_ns); /** * kbase_csf_firmware_reset_mcu_core_pwroff_time - Reset the MCU shader Core power-off diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c index a8dc411e6884..d08686f5829b 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_cfg.c @@ -145,7 +145,7 @@ static ssize_t store_fw_cfg(struct kobject *kobj, struct attribute *attr, const cur_val = config->cur_val; if (cur_val == val) { spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return count; + return (ssize_t)count; } /* If configuration update cannot be performed with @@ -209,7 +209,7 @@ static ssize_t store_fw_cfg(struct kobject *kobj, struct attribute *attr, const return -EINVAL; } - return count; + return (ssize_t)count; } static const struct sysfs_ops fw_cfg_ops = { @@ -373,17 +373,17 @@ int kbase_csf_firmware_cfg_fw_wa_init(struct kbase_device *kbdev) if (entry_count == -EINVAL || entry_count == -ENODATA) return 0; - entry_bytes = entry_count * sizeof(u32); + entry_bytes = (size_t)entry_count * sizeof(u32); kbdev->csf.quirks_ext = kzalloc(entry_bytes, GFP_KERNEL); if (!kbdev->csf.quirks_ext) return -ENOMEM; ret = of_property_read_u32_array(kbdev->dev->of_node, "quirks-ext", kbdev->csf.quirks_ext, - entry_count); + (size_t)entry_count); if (ret == -EINVAL) ret = of_property_read_u32_array(kbdev->dev->of_node, "quirks_ext", - kbdev->csf.quirks_ext, entry_count); + kbdev->csf.quirks_ext, (size_t)entry_count); if (ret == -EINVAL || ret == -ENODATA) { /* This is unexpected since the property is already accessed for counting the number diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_log.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_log.c index b57121649966..1af173c0e0b9 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_log.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_log.c @@ -144,7 +144,7 @@ static ssize_t kbasep_csf_firmware_log_debugfs_read(struct file *file, char __us } *ppos += n_read; - ret = n_read; + ret = (int)n_read; out: atomic_set(&fw_log->busy, 0); @@ -178,8 +178,9 @@ static int kbase_csf_firmware_log_mode_write(void *data, u64 val) break; case KBASE_CSF_FIRMWARE_LOG_MODE_AUTO_PRINT: case KBASE_CSF_FIRMWARE_LOG_MODE_AUTO_DISCARD: - schedule_delayed_work(&fw_log->poll_work, - msecs_to_jiffies(atomic_read(&fw_log->poll_period_ms))); + schedule_delayed_work( + &fw_log->poll_work, + msecs_to_jiffies((unsigned int)atomic_read(&fw_log->poll_period_ms))); break; default: ret = -EINVAL; @@ -198,7 +199,7 @@ static int kbase_csf_firmware_log_poll_period_read(void *data, u64 *val) struct kbase_device *kbdev = (struct kbase_device *)data; struct kbase_csf_firmware_log *fw_log = &kbdev->csf.fw_log; - *val = atomic_read(&fw_log->poll_period_ms); + *val = (u64)atomic_read(&fw_log->poll_period_ms); return 0; } @@ -263,7 +264,7 @@ static void kbase_csf_firmware_log_poll(struct work_struct *work) return; schedule_delayed_work(&fw_log->poll_work, - msecs_to_jiffies(atomic_read(&fw_log->poll_period_ms))); + msecs_to_jiffies((unsigned int)atomic_read(&fw_log->poll_period_ms))); } int kbase_csf_firmware_log_init(struct kbase_device *kbdev) @@ -382,7 +383,7 @@ void kbase_csf_firmware_log_dump_buffer(struct kbase_device *kbdev) pend = p + read_size; p = buf; - while (p < pend && (pnewline = memchr(p, '\n', pend - p))) { + while (p < pend && (pnewline = memchr(p, '\n', (size_t)(pend - p)))) { /* Null-terminate the string */ *pnewline = 0; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c index 346a28ee6772..90568f6fa09f 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c @@ -533,9 +533,9 @@ EXPORT_SYMBOL(kbase_csf_ring_doorbell); */ static void handle_internal_firmware_fatal(struct kbase_device *const kbdev) { - int as; + size_t as; - for (as = 0; as < kbdev->nr_hw_address_spaces; as++) { + for (as = 0; as < (size_t)kbdev->nr_hw_address_spaces; as++) { unsigned long flags; struct kbase_context *kctx; struct kbase_fault fault; @@ -604,7 +604,8 @@ static bool global_request_complete(struct kbase_device *const kbdev, u32 const static int wait_for_global_request(struct kbase_device *const kbdev, u32 const req_mask) { - const long wait_timeout = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); + const long wait_timeout = + kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); long remaining; int err = 0; @@ -893,12 +894,12 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) kbase_pm_update_state(kbdev); } -static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_ms, u32 *modifier) +static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_ns, u32 *no_modifier) { #define HYSTERESIS_VAL_UNIT_SHIFT (10) /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ - u64 freq = arch_timer_get_cntfrq(); - u64 dur_val = dur_ms; + u64 freq = kbase_arch_timer_get_cntfrq(kbdev); + u64 dur_val = dur_ns; u32 cnt_val_u32, reg_val_u32; bool src_system_timestamp = freq > 0; @@ -916,45 +917,46 @@ static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_m "Can't get the timestamp frequency, use cycle counter format with firmware idle hysteresis!"); } - /* Formula for dur_val = ((dur_ms/1000) * freq_HZ) >> 10) */ - dur_val = (dur_val * freq) >> HYSTERESIS_VAL_UNIT_SHIFT; - dur_val = div_u64(dur_val, 1000); - - *modifier = 0; + /* Formula for dur_val = (dur/1e9) * freq_HZ) */ + dur_val = dur_val * freq; + dur_val = div_u64(dur_val, NSEC_PER_SEC); + if (dur_val < S32_MAX) { + *no_modifier = 1; + } else { + dur_val = dur_val >> HYSTERESIS_VAL_UNIT_SHIFT; + *no_modifier = 0; + } /* Interface limits the value field to S32_MAX */ cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val; reg_val_u32 = GLB_IDLE_TIMER_TIMEOUT_SET(0, cnt_val_u32); /* add the source flag */ - if (src_system_timestamp) - reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET( - reg_val_u32, GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); - else - reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET( - reg_val_u32, GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER); + reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET( + reg_val_u32, (src_system_timestamp ? GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP : + GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER)); return reg_val_u32; } -u32 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev) +u64 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev) { unsigned long flags; - u32 dur; + u64 dur_ns; kbase_csf_scheduler_spin_lock(kbdev, &flags); - dur = kbdev->csf.gpu_idle_hysteresis_ns; + dur_ns = kbdev->csf.gpu_idle_hysteresis_ns; kbase_csf_scheduler_spin_unlock(kbdev, flags); - return dur; + return dur_ns; } -u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u32 dur) +u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u64 dur_ns) { unsigned long flags; - u32 modifier = 0; + u32 no_modifier = 0; - const u32 hysteresis_val = convert_dur_to_idle_count(kbdev, dur, &modifier); + const u32 hysteresis_val = convert_dur_to_idle_count(kbdev, dur_ns, &no_modifier); /* The 'fw_load_lock' is taken to synchronize against the deferred * loading of FW, where the idle timer will be enabled. @@ -962,9 +964,9 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, mutex_lock(&kbdev->fw_load_lock); if (unlikely(!kbdev->csf.firmware_inited)) { kbase_csf_scheduler_spin_lock(kbdev, &flags); - kbdev->csf.gpu_idle_hysteresis_ns = dur; + kbdev->csf.gpu_idle_hysteresis_ns = dur_ns; kbdev->csf.gpu_idle_dur_count = hysteresis_val; - kbdev->csf.gpu_idle_dur_count_modifier = modifier; + kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; kbase_csf_scheduler_spin_unlock(kbdev, flags); mutex_unlock(&kbdev->fw_load_lock); goto end; @@ -1002,9 +1004,9 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK); kbase_csf_scheduler_spin_lock(kbdev, &flags); - kbdev->csf.gpu_idle_hysteresis_ns = dur; + kbdev->csf.gpu_idle_hysteresis_ns = dur_ns; kbdev->csf.gpu_idle_dur_count = hysteresis_val; - kbdev->csf.gpu_idle_dur_count_modifier = modifier; + kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; kbase_csf_firmware_enable_gpu_idle_timer(kbdev); kbase_csf_scheduler_spin_unlock(kbdev, flags); wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK); @@ -1018,12 +1020,12 @@ end: return hysteresis_val; } -static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u32 dur_us, - u32 *modifier) +static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u64 dur_ns, + u32 *no_modifier) { /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ - u64 freq = arch_timer_get_cntfrq(); - u64 dur_val = dur_us; + u64 freq = kbase_arch_timer_get_cntfrq(kbdev); + u64 dur_val = dur_ns; u32 cnt_val_u32, reg_val_u32; bool src_system_timestamp = freq > 0; @@ -1041,50 +1043,57 @@ static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u3 "Can't get the timestamp frequency, use cycle counter with MCU shader Core Poweroff timer!"); } - /* Formula for dur_val = ((dur_us/1e6) * freq_HZ) >> 10) */ - dur_val = (dur_val * freq) >> HYSTERESIS_VAL_UNIT_SHIFT; - dur_val = div_u64(dur_val, 1000000); - - *modifier = 0; + /* Formula for dur_val = (dur/1e9) * freq_HZ) */ + dur_val = dur_val * freq; + dur_val = div_u64(dur_val, NSEC_PER_SEC); + if (dur_val < S32_MAX) { + *no_modifier = 1; + } else { + dur_val = dur_val >> HYSTERESIS_VAL_UNIT_SHIFT; + *no_modifier = 0; + } /* Interface limits the value field to S32_MAX */ - cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val; + if (dur_val > S32_MAX) { + /* Upper Bound - as interface limits the field to S32_MAX */ + cnt_val_u32 = S32_MAX; + } else { + cnt_val_u32 = (u32)dur_val; + } reg_val_u32 = GLB_PWROFF_TIMER_TIMEOUT_SET(0, cnt_val_u32); /* add the source flag */ - if (src_system_timestamp) - reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET( - reg_val_u32, GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); - else - reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET( - reg_val_u32, GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER); + reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET( + reg_val_u32, + (src_system_timestamp ? GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP : + GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER)); return reg_val_u32; } -u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev) +u64 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev) { - u32 pwroff; + u64 pwroff_ns; unsigned long flags; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - pwroff = kbdev->csf.mcu_core_pwroff_dur_ns; + pwroff_ns = kbdev->csf.mcu_core_pwroff_dur_ns; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return pwroff; + return pwroff_ns; } -u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32 dur) +u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u64 dur_ns) { unsigned long flags; - u32 modifier = 0; + u32 no_modifier = 0; - const u32 pwroff = convert_dur_to_core_pwroff_count(kbdev, dur, &modifier); + const u32 pwroff = convert_dur_to_core_pwroff_count(kbdev, dur_ns, &no_modifier); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbdev->csf.mcu_core_pwroff_dur_ns = dur; + kbdev->csf.mcu_core_pwroff_dur_ns = dur_ns; kbdev->csf.mcu_core_pwroff_dur_count = pwroff; - kbdev->csf.mcu_core_pwroff_dur_count_modifier = modifier; + kbdev->csf.mcu_core_pwroff_dur_count_no_modifier = no_modifier; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); dev_dbg(kbdev->dev, "MCU shader Core Poweroff input update: 0x%.8x", pwroff); @@ -1101,8 +1110,6 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev) { init_waitqueue_head(&kbdev->csf.event_wait); - kbdev->csf.fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); - kbase_csf_firmware_reset_mcu_core_pwroff_time(kbdev); INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces); INIT_LIST_HEAD(&kbdev->csf.firmware_config); @@ -1125,7 +1132,7 @@ void kbase_csf_firmware_early_term(struct kbase_device *kbdev) int kbase_csf_firmware_late_init(struct kbase_device *kbdev) { - u32 modifier = 0; + u32 no_modifier = 0; kbdev->csf.gpu_idle_hysteresis_ns = FIRMWARE_IDLE_HYSTERESIS_TIME_NS; #ifdef KBASE_PM_RUNTIME @@ -1134,8 +1141,8 @@ int kbase_csf_firmware_late_init(struct kbase_device *kbdev) #endif WARN_ON(!kbdev->csf.gpu_idle_hysteresis_ns); kbdev->csf.gpu_idle_dur_count = - convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &modifier); - kbdev->csf.gpu_idle_dur_count_modifier = modifier; + convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &no_modifier); + kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; return 0; } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c index 9da7f4dea64b..76e42e847fc3 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c @@ -464,7 +464,9 @@ static void kbase_kcpu_jit_retry_pending_allocs(struct kbase_context *kctx) * kbase_csf_kcpu_queue_context.jit_lock . */ list_for_each_entry(blocked_queue, &kctx->csf.kcpu_queues.jit_blocked_queues, jit_blocked) - queue_work(blocked_queue->wq, &blocked_queue->work); + queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio : + kctx->csf.kcpu_wq_normal_prio, + &blocked_queue->work); } static int kbase_kcpu_jit_free_process(struct kbase_kcpu_command_queue *queue, @@ -510,7 +512,7 @@ static int kbase_kcpu_jit_free_process(struct kbase_kcpu_command_queue *queue, } KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( - queue->kctx->kbdev, queue, item_err, pages_used); + queue->kctx->kbdev, queue, (u32)item_err, pages_used); } /* @@ -611,7 +613,7 @@ kbase_csf_queue_group_suspend_prepare(struct kbase_kcpu_command_queue *kcpu_queu u64 page_addr = addr & PAGE_MASK; u64 end_addr = addr + csg_suspend_buf_size - 1; u64 last_page_addr = end_addr & PAGE_MASK; - int nr_pages = (last_page_addr - page_addr) / PAGE_SIZE + 1; + unsigned int nr_pages = (last_page_addr - page_addr) / PAGE_SIZE + 1; int pinned_pages = 0, ret = 0; struct kbase_va_region *reg; @@ -647,7 +649,7 @@ kbase_csf_queue_group_suspend_prepare(struct kbase_kcpu_command_queue *kcpu_queu if (kbase_is_region_invalid_or_free(reg)) { kbase_gpu_vm_unlock(kctx); - pinned_pages = get_user_pages_fast(page_addr, nr_pages, 1, sus_buf->pages); + pinned_pages = get_user_pages_fast(page_addr, (int)nr_pages, 1, sus_buf->pages); kbase_gpu_vm_lock(kctx); if (pinned_pages < 0) { @@ -715,8 +717,11 @@ static int kbase_csf_queue_group_suspend_process(struct kbase_context *kctx, static enum kbase_csf_event_callback_action event_cqs_callback(void *param) { struct kbase_kcpu_command_queue *kcpu_queue = (struct kbase_kcpu_command_queue *)param; + struct kbase_context *kctx = kcpu_queue->kctx; - queue_work(kcpu_queue->wq, &kcpu_queue->work); + queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio : + kctx->csf.kcpu_wq_normal_prio, + &kcpu_queue->work); return KBASE_CSF_EVENT_CALLBACK_KEEP; } @@ -1301,11 +1306,7 @@ static int kbase_kcpu_cqs_set_operation_prepare( } #if IS_ENABLED(CONFIG_SYNC_FILE) -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -static void kbase_csf_fence_wait_callback(struct fence *fence, struct fence_cb *cb) -#else static void kbase_csf_fence_wait_callback(struct dma_fence *fence, struct dma_fence_cb *cb) -#endif { struct kbase_kcpu_command_fence_info *fence_info = container_of(cb, struct kbase_kcpu_command_fence_info, fence_cb); @@ -1321,7 +1322,9 @@ static void kbase_csf_fence_wait_callback(struct dma_fence *fence, struct dma_fe fence->seqno); /* Resume kcpu command queue processing. */ - queue_work(kcpu_queue->wq, &kcpu_queue->work); + queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio : + kctx->csf.kcpu_wq_normal_prio, + &kcpu_queue->work); } static void kbasep_kcpu_fence_wait_cancel(struct kbase_kcpu_command_queue *kcpu_queue, @@ -1374,11 +1377,7 @@ static void fence_timeout_callback(struct timer_list *timer) struct kbase_context *const kctx = kcpu_queue->kctx; struct kbase_kcpu_command *cmd = &kcpu_queue->commands[kcpu_queue->start_offset]; struct kbase_kcpu_command_fence_info *fence_info; -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; -#else struct dma_fence *fence; -#endif struct kbase_sync_fence_info info; if (cmd->type != BASE_KCPU_COMMAND_TYPE_FENCE_WAIT) { @@ -1400,7 +1399,9 @@ static void fence_timeout_callback(struct timer_list *timer) kbase_sync_fence_info_get(fence, &info); if (info.status == 1) { - queue_work(kcpu_queue->wq, &kcpu_queue->work); + queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio : + kctx->csf.kcpu_wq_normal_prio, + &kcpu_queue->work); } else if (info.status == 0) { dev_warn(kctx->kbdev->dev, "fence has not yet signalled in %ums", FENCE_WAIT_TIMEOUT_MS); @@ -1444,11 +1445,7 @@ static int kbase_kcpu_fence_wait_process(struct kbase_kcpu_command_queue *kcpu_q struct kbase_kcpu_command_fence_info *fence_info) { int fence_status = 0; -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; -#else struct dma_fence *fence; -#endif struct kbase_context *const kctx = kcpu_queue->kctx; lockdep_assert_held(&kcpu_queue->lock); @@ -1514,11 +1511,7 @@ static int kbase_kcpu_fence_wait_prepare(struct kbase_kcpu_command_queue *kcpu_q struct base_kcpu_command_fence_info *fence_info, struct kbase_kcpu_command *current_command) { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence_in; -#else struct dma_fence *fence_in; -#endif struct base_fence fence; lockdep_assert_held(&kcpu_queue->lock); @@ -1613,11 +1606,7 @@ static void kcpu_force_signal_fence(struct kbase_kcpu_command_queue *kcpu_queue) { int status; int i; -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; -#else struct dma_fence *fence; -#endif struct kbase_context *const kctx = kcpu_queue->kctx; #ifdef CONFIG_MALI_BIFROST_FENCE_DEBUG int del; @@ -1703,7 +1692,9 @@ static void fence_signal_timeout_cb(struct timer_list *timer) if (atomic_read(&kcpu_queue->fence_signal_pending_cnt) > 1) fence_signal_timeout_start(kcpu_queue); - queue_work(kcpu_queue->wq, &kcpu_queue->timeout_work); + queue_work(atomic_read(&kctx->prioritized) ? kctx->csf.kcpu_wq_high_prio : + kctx->csf.kcpu_wq_normal_prio, + &kcpu_queue->timeout_work); } } @@ -1762,11 +1753,7 @@ static int kbasep_kcpu_fence_signal_init(struct kbase_kcpu_command_queue *kcpu_q struct base_fence *fence, struct sync_file **sync_file, int *fd) { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence_out; -#else struct dma_fence *fence_out; -#endif struct kbase_kcpu_dma_fence *kcpu_fence; int ret = 0; @@ -1780,11 +1767,7 @@ static int kbasep_kcpu_fence_signal_init(struct kbase_kcpu_command_queue *kcpu_q kcpu_fence->metadata = kcpu_queue->metadata; WARN_ON(!kbase_refcount_inc_not_zero(&kcpu_fence->metadata->refcount)); -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - fence_out = (struct fence *)kcpu_fence; -#else fence_out = (struct dma_fence *)kcpu_fence; -#endif dma_fence_init(fence_out, &kbase_fence_ops, &kbase_csf_fence_lock, kcpu_queue->fence_context, ++kcpu_queue->fence_seqno); @@ -1859,7 +1842,7 @@ static int kbase_kcpu_fence_signal_prepare(struct kbase_kcpu_command_queue *kcpu * installed, so the install step needs to be done at the last * before returning success. */ - fd_install(fd, sync_file->file); + fd_install((unsigned int)fd, sync_file->file); if (atomic_inc_return(&kcpu_queue->fence_signal_pending_cnt) == 1) fence_signal_timeout_start(kcpu_queue); @@ -1903,11 +1886,7 @@ static void kcpu_fence_timeout_dump(struct kbase_kcpu_command_queue *queue, struct kbase_kcpu_command *cmd; struct kbase_kcpu_command_fence_info *fence_info; struct kbase_kcpu_dma_fence *kcpu_fence; -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; -#else struct dma_fence *fence; -#endif struct kbase_sync_fence_info info; u16 i; @@ -2045,8 +2024,6 @@ static int delete_queue(struct kbase_context *kctx, u32 id) cancel_work_sync(&queue->timeout_work); cancel_work_sync(&queue->work); - destroy_workqueue(queue->wq); - mutex_destroy(&queue->lock); kfree(queue); @@ -2085,7 +2062,7 @@ static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_INFO( #endif /* CONFIG_MALI_VECTOR_DUMP */ } KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( - kbdev, queue, alloc_status, gpu_alloc_addr, mmu_flags); + kbdev, queue, (u32)alloc_status, gpu_alloc_addr, mmu_flags); } } @@ -2584,7 +2561,7 @@ int kbase_csf_kcpu_queue_enqueue(struct kbase_context *kctx, } } - kcpu_cmd->enqueue_ts = atomic64_inc_return(&kctx->csf.kcpu_queues.cmd_seq_num); + kcpu_cmd->enqueue_ts = (u64)atomic64_inc_return(&kctx->csf.kcpu_queues.cmd_seq_num); switch (command.type) { case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: #if IS_ENABLED(CONFIG_SYNC_FILE) @@ -2676,6 +2653,23 @@ out: int kbase_csf_kcpu_queue_context_init(struct kbase_context *kctx) { + kctx->csf.kcpu_wq_high_prio = alloc_workqueue("mali_kcpu_wq_%i_high_prio", + WQ_UNBOUND | WQ_HIGHPRI, 0, kctx->tgid); + if (kctx->csf.kcpu_wq_high_prio == NULL) { + dev_err(kctx->kbdev->dev, + "Failed to initialize KCPU queue high-priority workqueue"); + return -ENOMEM; + } + + kctx->csf.kcpu_wq_normal_prio = + alloc_workqueue("mali_kcpu_wq_%i_normal_prio", 0, 0, kctx->tgid); + if (kctx->csf.kcpu_wq_normal_prio == NULL) { + dev_err(kctx->kbdev->dev, + "Failed to initialize KCPU queue normal-priority workqueue"); + destroy_workqueue(kctx->csf.kcpu_wq_high_prio); + return -ENOMEM; + } + mutex_init(&kctx->csf.kcpu_queues.lock); return 0; @@ -2689,10 +2683,13 @@ void kbase_csf_kcpu_queue_context_term(struct kbase_context *kctx) if (WARN_ON(!kctx->csf.kcpu_queues.array[id])) clear_bit(id, kctx->csf.kcpu_queues.in_use); else - (void)delete_queue(kctx, id); + (void)delete_queue(kctx, (u32)id); } mutex_destroy(&kctx->csf.kcpu_queues.lock); + + destroy_workqueue(kctx->csf.kcpu_wq_normal_prio); + destroy_workqueue(kctx->csf.kcpu_wq_high_prio); } KBASE_EXPORT_TEST_API(kbase_csf_kcpu_queue_context_term); @@ -2737,15 +2734,7 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu goto out; } - queue->wq = alloc_workqueue("mali_kbase_csf_kcpu_wq_%i", WQ_UNBOUND | WQ_HIGHPRI, 0, idx); - if (queue->wq == NULL) { - kfree(queue); - ret = -ENOMEM; - - goto out; - } - - bitmap_set(kctx->csf.kcpu_queues.in_use, idx, 1); + bitmap_set(kctx->csf.kcpu_queues.in_use, (unsigned int)idx, 1); kctx->csf.kcpu_queues.array[idx] = queue; mutex_init(&queue->lock); queue->kctx = kctx; @@ -2758,7 +2747,6 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu metadata = kzalloc(sizeof(*metadata), GFP_KERNEL); if (!metadata) { - destroy_workqueue(queue->wq); kfree(queue); ret = -ENOMEM; goto out; @@ -2766,10 +2754,9 @@ int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, struct kbase_ioctl_kcpu metadata->kbdev = kctx->kbdev; metadata->kctx_id = kctx->id; - n = snprintf(metadata->timeline_name, MAX_TIMELINE_NAME, "%d-%d_%d-%lld-kcpu", + n = snprintf(metadata->timeline_name, MAX_TIMELINE_NAME, "%u-%d_%u-%llu-kcpu", kctx->kbdev->id, kctx->tgid, kctx->id, queue->fence_context); if (WARN_ON(n >= MAX_TIMELINE_NAME)) { - destroy_workqueue(queue->wq); kfree(queue); kfree(metadata); ret = -EINVAL; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h index 9ca33773941e..d1f18ed5caca 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.h @@ -25,11 +25,7 @@ #include #include -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -#include -#else -#include -#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ +#include /* The maximum number of KCPU commands in flight, enqueueing more commands * than this value shall block. @@ -56,13 +52,8 @@ struct kbase_kcpu_command_import_info { * @fence_has_force_signaled: fence has forced signaled after fence timeouted */ struct kbase_kcpu_command_fence_info { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence_cb fence_cb; - struct fence *fence; -#else struct dma_fence_cb fence_cb; struct dma_fence *fence; -#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ struct kbase_kcpu_command_queue *kcpu_queue; bool fence_has_force_signaled; }; @@ -249,7 +240,6 @@ struct kbase_kcpu_command { * @kctx: The context to which this command queue belongs. * @commands: Array of commands which have been successfully * enqueued to this command queue. - * @wq: Dedicated workqueue for processing commands. * @work: struct work_struct which contains a pointer to * the function which handles processing of kcpu * commands enqueued into a kcpu command queue; @@ -296,7 +286,6 @@ struct kbase_kcpu_command_queue { struct mutex lock; struct kbase_context *kctx; struct kbase_kcpu_command commands[KBASEP_KCPU_QUEUE_SIZE]; - struct workqueue_struct *wq; struct work_struct work; struct work_struct timeout_work; u8 start_offset; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_fence_debugfs.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_fence_debugfs.c index 9cbf7bab5caa..1433aae613b4 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_fence_debugfs.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu_fence_debugfs.c @@ -60,9 +60,9 @@ static ssize_t kbase_csf_kcpu_queue_fence_signal_enabled_set(struct file *file, if (ret < 0) return ret; - atomic_set(&kbdev->fence_signal_timeout_enabled, enabled); + atomic_set(&kbdev->fence_signal_timeout_enabled, (int)enabled); - return count; + return (ssize_t)count; } static const struct file_operations kbase_csf_kcpu_queue_fence_signal_fops = { @@ -82,7 +82,7 @@ static ssize_t kbase_csf_kcpu_queue_fence_signal_timeout_get(struct file *file, unsigned int timeout_ms = kbase_get_timeout_ms(kbdev, KCPU_FENCE_SIGNAL_TIMEOUT); size = scnprintf(buffer, sizeof(buffer), "%u\n", timeout_ms); - return simple_read_from_buffer(buf, count, ppos, buffer, size); + return simple_read_from_buffer(buf, count, ppos, buffer, (size_t)size); } static ssize_t kbase_csf_kcpu_queue_fence_signal_timeout_set(struct file *file, @@ -105,7 +105,7 @@ static ssize_t kbase_csf_kcpu_queue_fence_signal_timeout_set(struct file *file, */ kbase_device_set_timeout_ms(kbdev, KCPU_FENCE_SIGNAL_TIMEOUT, timeout_ms); - return count; + return (ssize_t)count; } static const struct file_operations kbase_csf_kcpu_queue_fence_signal_timeout_fops = { diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_mcu_shared_reg.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_mcu_shared_reg.c index 11c0ba499596..c8cf17f08e6f 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_mcu_shared_reg.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_mcu_shared_reg.c @@ -412,7 +412,7 @@ int kbase_csf_mcu_shared_add_queue(struct kbase_device *kbdev, struct kbase_queu "No bound csg_reg, or in wrong state")) return -EIO; - vpfn = CSG_REG_USERIO_VPFN(csg_reg->reg, queue->csi_index, nr_susp_pages); + vpfn = CSG_REG_USERIO_VPFN(csg_reg->reg, (u32)queue->csi_index, nr_susp_pages); err = userio_pages_replace_phys(kbdev, vpfn, queue->phys); if (likely(!err)) { /* Mark the queue has been successfully mapped */ @@ -452,7 +452,7 @@ void kbase_csf_mcu_shared_drop_stopped_queue(struct kbase_device *kbdev, struct csg_reg = get_group_bound_csg_reg(group); - vpfn = CSG_REG_USERIO_VPFN(csg_reg->reg, queue->csi_index, nr_susp_pages); + vpfn = CSG_REG_USERIO_VPFN(csg_reg->reg, (u32)queue->csi_index, nr_susp_pages); WARN_ONCE(userio_pages_replace_phys(kbdev, vpfn, NULL), "Unexpected restoring to dummy map update error"); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c index e78144ac4bf3..c646e785e89a 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_protected_memory.c @@ -79,7 +79,7 @@ kbase_csf_protected_memory_alloc(struct kbase_device *const kbdev, struct tagged unsigned int num_pages_order; if (is_small_page) - order = KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER; + order = KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER; num_pages_order = (1u << order); @@ -135,7 +135,7 @@ void kbase_csf_protected_memory_free(struct kbase_device *const kbdev, unsigned int num_pages_order = (1u << KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER); if (is_small_page) - num_pages_order = (1u << KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER); + num_pages_order = (1u << KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER); if (WARN_ON(!pma_dev) || WARN_ON(!pma)) return; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h index c4e6e4ac0df7..d01f3070cf5b 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h @@ -271,21 +271,21 @@ /* CS_KERNEL_INPUT_BLOCK register set definitions */ /* GLB_VERSION register */ #define GLB_VERSION_PATCH_SHIFT (0) -#define GLB_VERSION_PATCH_MASK ((0xFFFF) << GLB_VERSION_PATCH_SHIFT) +#define GLB_VERSION_PATCH_MASK ((0xFFFFU) << GLB_VERSION_PATCH_SHIFT) #define GLB_VERSION_PATCH_GET(reg_val) \ (((reg_val)&GLB_VERSION_PATCH_MASK) >> GLB_VERSION_PATCH_SHIFT) #define GLB_VERSION_PATCH_SET(reg_val, value) \ (((reg_val) & ~GLB_VERSION_PATCH_MASK) | \ (((value) << GLB_VERSION_PATCH_SHIFT) & GLB_VERSION_PATCH_MASK)) #define GLB_VERSION_MINOR_SHIFT (16) -#define GLB_VERSION_MINOR_MASK ((0xFF) << GLB_VERSION_MINOR_SHIFT) +#define GLB_VERSION_MINOR_MASK ((0xFFU) << GLB_VERSION_MINOR_SHIFT) #define GLB_VERSION_MINOR_GET(reg_val) \ (((reg_val)&GLB_VERSION_MINOR_MASK) >> GLB_VERSION_MINOR_SHIFT) #define GLB_VERSION_MINOR_SET(reg_val, value) \ (((reg_val) & ~GLB_VERSION_MINOR_MASK) | \ (((value) << GLB_VERSION_MINOR_SHIFT) & GLB_VERSION_MINOR_MASK)) #define GLB_VERSION_MAJOR_SHIFT (24) -#define GLB_VERSION_MAJOR_MASK ((0xFF) << GLB_VERSION_MAJOR_SHIFT) +#define GLB_VERSION_MAJOR_MASK ((0xFFU) << GLB_VERSION_MAJOR_SHIFT) #define GLB_VERSION_MAJOR_GET(reg_val) \ (((reg_val)&GLB_VERSION_MAJOR_MASK) >> GLB_VERSION_MAJOR_SHIFT) #define GLB_VERSION_MAJOR_SET(reg_val, value) \ @@ -311,60 +311,60 @@ (((value) << CS_REQ_EXTRACT_EVENT_SHIFT) & CS_REQ_EXTRACT_EVENT_MASK)) #define CS_REQ_IDLE_SYNC_WAIT_SHIFT 8 -#define CS_REQ_IDLE_SYNC_WAIT_MASK (0x1 << CS_REQ_IDLE_SYNC_WAIT_SHIFT) +#define CS_REQ_IDLE_SYNC_WAIT_MASK (0x1U << CS_REQ_IDLE_SYNC_WAIT_SHIFT) #define CS_REQ_IDLE_SYNC_WAIT_GET(reg_val) \ (((reg_val)&CS_REQ_IDLE_SYNC_WAIT_MASK) >> CS_REQ_IDLE_SYNC_WAIT_SHIFT) #define CS_REQ_IDLE_SYNC_WAIT_SET(reg_val, value) \ (((reg_val) & ~CS_REQ_IDLE_SYNC_WAIT_MASK) | \ (((value) << CS_REQ_IDLE_SYNC_WAIT_SHIFT) & CS_REQ_IDLE_SYNC_WAIT_MASK)) #define CS_REQ_IDLE_PROTM_PEND_SHIFT 9 -#define CS_REQ_IDLE_PROTM_PEND_MASK (0x1 << CS_REQ_IDLE_PROTM_PEND_SHIFT) +#define CS_REQ_IDLE_PROTM_PEND_MASK (0x1U << CS_REQ_IDLE_PROTM_PEND_SHIFT) #define CS_REQ_IDLE_PROTM_PEND_GET(reg_val) \ (((reg_val)&CS_REQ_IDLE_PROTM_PEND_MASK) >> CS_REQ_IDLE_PROTM_PEND_SHIFT) #define CS_REQ_IDLE_PROTM_PEND_SET(reg_val, value) \ (((reg_val) & ~CS_REQ_IDLE_PROTM_PEND_MASK) | \ (((value) << CS_REQ_IDLE_PROTM_PEND_SHIFT) & CS_REQ_IDLE_PROTM_PEND_MASK)) #define CS_REQ_IDLE_EMPTY_SHIFT 10 -#define CS_REQ_IDLE_EMPTY_MASK (0x1 << CS_REQ_IDLE_EMPTY_SHIFT) +#define CS_REQ_IDLE_EMPTY_MASK (0x1U << CS_REQ_IDLE_EMPTY_SHIFT) #define CS_REQ_IDLE_EMPTY_GET(reg_val) \ (((reg_val)&CS_REQ_IDLE_EMPTY_MASK) >> CS_REQ_IDLE_EMPTY_SHIFT) #define CS_REQ_IDLE_EMPTY_SET(reg_val, value) \ (((reg_val) & ~CS_REQ_IDLE_EMPTY_MASK) | \ (((value) << CS_REQ_IDLE_EMPTY_SHIFT) & CS_REQ_IDLE_EMPTY_MASK)) #define CS_REQ_IDLE_RESOURCE_REQ_SHIFT 11 -#define CS_REQ_IDLE_RESOURCE_REQ_MASK (0x1 << CS_REQ_IDLE_RESOURCE_REQ_SHIFT) +#define CS_REQ_IDLE_RESOURCE_REQ_MASK (0x1U << CS_REQ_IDLE_RESOURCE_REQ_SHIFT) #define CS_REQ_IDLE_RESOURCE_REQ_GET(reg_val) \ (((reg_val)&CS_REQ_IDLE_RESOURCE_REQ_MASK) >> CS_REQ_IDLE_RESOURCE_REQ_SHIFT) #define CS_REQ_IDLE_RESOURCE_REQ_SET(reg_val, value) \ (((reg_val) & ~CS_REQ_IDLE_RESOURCE_REQ_MASK) | \ (((value) << CS_REQ_IDLE_RESOURCE_REQ_SHIFT) & CS_REQ_IDLE_RESOURCE_REQ_MASK)) #define CS_REQ_IDLE_SHARED_SB_DEC_SHIFT 12 -#define CS_REQ_IDLE_SHARED_SB_DEC_MASK (0x1 << CS_REQ_IDLE_SHARED_SB_DEC_SHIFT) +#define CS_REQ_IDLE_SHARED_SB_DEC_MASK (0x1U << CS_REQ_IDLE_SHARED_SB_DEC_SHIFT) #define CS_REQ_IDLE_SHARED_SB_DEC_GET(reg_val) \ (((reg_val)&CS_REQ_IDLE_SHARED_SB_DEC_MASK) >> CS_REQ_IDLE_SHARED_SB_DEC_SHIFT) #define CS_REQ_IDLE_SHARED_SB_DEC_REQ_SET(reg_val, value) \ (((reg_val) & ~CS_REQ_IDLE_SHARED_SB_DEC_MASK) | \ (((value) << CS_REQ_IDLE_SHARED_SB_DEC_SHIFT) & CS_REQ_IDLE_SHARED_SB_DEC_MASK)) #define CS_REQ_TILER_OOM_SHIFT 26 -#define CS_REQ_TILER_OOM_MASK (0x1 << CS_REQ_TILER_OOM_SHIFT) +#define CS_REQ_TILER_OOM_MASK (0x1U << CS_REQ_TILER_OOM_SHIFT) #define CS_REQ_TILER_OOM_GET(reg_val) (((reg_val)&CS_REQ_TILER_OOM_MASK) >> CS_REQ_TILER_OOM_SHIFT) #define CS_REQ_TILER_OOM_SET(reg_val, value) \ (((reg_val) & ~CS_REQ_TILER_OOM_MASK) | \ (((value) << CS_REQ_TILER_OOM_SHIFT) & CS_REQ_TILER_OOM_MASK)) #define CS_REQ_PROTM_PEND_SHIFT 27 -#define CS_REQ_PROTM_PEND_MASK (0x1 << CS_REQ_PROTM_PEND_SHIFT) +#define CS_REQ_PROTM_PEND_MASK (0x1U << CS_REQ_PROTM_PEND_SHIFT) #define CS_REQ_PROTM_PEND_GET(reg_val) \ (((reg_val)&CS_REQ_PROTM_PEND_MASK) >> CS_REQ_PROTM_PEND_SHIFT) #define CS_REQ_PROTM_PEND_SET(reg_val, value) \ (((reg_val) & ~CS_REQ_PROTM_PEND_MASK) | \ (((value) << CS_REQ_PROTM_PEND_SHIFT) & CS_REQ_PROTM_PEND_MASK)) #define CS_REQ_FATAL_SHIFT 30 -#define CS_REQ_FATAL_MASK (0x1 << CS_REQ_FATAL_SHIFT) +#define CS_REQ_FATAL_MASK (0x1U << CS_REQ_FATAL_SHIFT) #define CS_REQ_FATAL_GET(reg_val) (((reg_val)&CS_REQ_FATAL_MASK) >> CS_REQ_FATAL_SHIFT) #define CS_REQ_FATAL_SET(reg_val, value) \ (((reg_val) & ~CS_REQ_FATAL_MASK) | (((value) << CS_REQ_FATAL_SHIFT) & CS_REQ_FATAL_MASK)) #define CS_REQ_FAULT_SHIFT 31 -#define CS_REQ_FAULT_MASK (0x1 << CS_REQ_FAULT_SHIFT) +#define CS_REQ_FAULT_MASK (0x1U << CS_REQ_FAULT_SHIFT) #define CS_REQ_FAULT_GET(reg_val) (((reg_val)&CS_REQ_FAULT_MASK) >> CS_REQ_FAULT_SHIFT) #define CS_REQ_FAULT_SET(reg_val, value) \ (((reg_val) & ~CS_REQ_FAULT_MASK) | (((value) << CS_REQ_FAULT_SHIFT) & CS_REQ_FAULT_MASK)) @@ -559,32 +559,32 @@ #define CS_ACK_STATE_START 0x1 /* End of CS_ACK_STATE values */ #define CS_ACK_EXTRACT_EVENT_SHIFT 4 -#define CS_ACK_EXTRACT_EVENT_MASK (0x1 << CS_ACK_EXTRACT_EVENT_SHIFT) +#define CS_ACK_EXTRACT_EVENT_MASK (0x1U << CS_ACK_EXTRACT_EVENT_SHIFT) #define CS_ACK_EXTRACT_EVENT_GET(reg_val) \ (((reg_val)&CS_ACK_EXTRACT_EVENT_MASK) >> CS_ACK_EXTRACT_EVENT_SHIFT) #define CS_ACK_EXTRACT_EVENT_SET(reg_val, value) \ (((reg_val) & ~CS_ACK_EXTRACT_EVENT_MASK) | \ (((value) << CS_ACK_EXTRACT_EVENT_SHIFT) & CS_ACK_EXTRACT_EVENT_MASK)) #define CS_ACK_TILER_OOM_SHIFT 26 -#define CS_ACK_TILER_OOM_MASK (0x1 << CS_ACK_TILER_OOM_SHIFT) +#define CS_ACK_TILER_OOM_MASK (0x1U << CS_ACK_TILER_OOM_SHIFT) #define CS_ACK_TILER_OOM_GET(reg_val) (((reg_val)&CS_ACK_TILER_OOM_MASK) >> CS_ACK_TILER_OOM_SHIFT) #define CS_ACK_TILER_OOM_SET(reg_val, value) \ (((reg_val) & ~CS_ACK_TILER_OOM_MASK) | \ (((value) << CS_ACK_TILER_OOM_SHIFT) & CS_ACK_TILER_OOM_MASK)) #define CS_ACK_PROTM_PEND_SHIFT 27 -#define CS_ACK_PROTM_PEND_MASK (0x1 << CS_ACK_PROTM_PEND_SHIFT) +#define CS_ACK_PROTM_PEND_MASK (0x1U << CS_ACK_PROTM_PEND_SHIFT) #define CS_ACK_PROTM_PEND_GET(reg_val) \ (((reg_val)&CS_ACK_PROTM_PEND_MASK) >> CS_ACK_PROTM_PEND_SHIFT) #define CS_ACK_PROTM_PEND_SET(reg_val, value) \ (((reg_val) & ~CS_ACK_PROTM_PEND_MASK) | \ (((value) << CS_ACK_PROTM_PEND_SHIFT) & CS_ACK_PROTM_PEND_MASK)) #define CS_ACK_FATAL_SHIFT 30 -#define CS_ACK_FATAL_MASK (0x1 << CS_ACK_FATAL_SHIFT) +#define CS_ACK_FATAL_MASK (0x1U << CS_ACK_FATAL_SHIFT) #define CS_ACK_FATAL_GET(reg_val) (((reg_val)&CS_ACK_FATAL_MASK) >> CS_ACK_FATAL_SHIFT) #define CS_ACK_FATAL_SET(reg_val, value) \ (((reg_val) & ~CS_ACK_FATAL_MASK) | (((value) << CS_ACK_FATAL_SHIFT) & CS_ACK_FATAL_MASK)) #define CS_ACK_FAULT_SHIFT 31 -#define CS_ACK_FAULT_MASK (0x1 << CS_ACK_FAULT_SHIFT) +#define CS_ACK_FAULT_MASK (0x1U << CS_ACK_FAULT_SHIFT) #define CS_ACK_FAULT_GET(reg_val) (((reg_val)&CS_ACK_FAULT_MASK) >> CS_ACK_FAULT_SHIFT) #define CS_ACK_FAULT_SET(reg_val, value) \ (((reg_val) & ~CS_ACK_FAULT_MASK) | (((value) << CS_ACK_FAULT_SHIFT) & CS_ACK_FAULT_MASK)) @@ -601,21 +601,21 @@ /* CS_STATUS_WAIT register */ #define CS_STATUS_WAIT_SB_MASK_SHIFT 0 -#define CS_STATUS_WAIT_SB_MASK_MASK (0xFFFF << CS_STATUS_WAIT_SB_MASK_SHIFT) +#define CS_STATUS_WAIT_SB_MASK_MASK (0xFFFFU << CS_STATUS_WAIT_SB_MASK_SHIFT) #define CS_STATUS_WAIT_SB_MASK_GET(reg_val) \ (((reg_val)&CS_STATUS_WAIT_SB_MASK_MASK) >> CS_STATUS_WAIT_SB_MASK_SHIFT) #define CS_STATUS_WAIT_SB_MASK_SET(reg_val, value) \ (((reg_val) & ~CS_STATUS_WAIT_SB_MASK_MASK) | \ (((value) << CS_STATUS_WAIT_SB_MASK_SHIFT) & CS_STATUS_WAIT_SB_MASK_MASK)) #define CS_STATUS_WAIT_SB_SOURCE_SHIFT 16 -#define CS_STATUS_WAIT_SB_SOURCE_MASK (0xF << CS_STATUS_WAIT_SB_SOURCE_SHIFT) +#define CS_STATUS_WAIT_SB_SOURCE_MASK (0xFU << CS_STATUS_WAIT_SB_SOURCE_SHIFT) #define CS_STATUS_WAIT_SB_SOURCE_GET(reg_val) \ (((reg_val)&CS_STATUS_WAIT_SB_SOURCE_MASK) >> CS_STATUS_WAIT_SB_SOURCE_SHIFT) #define CS_STATUS_WAIT_SB_SOURCE_SET(reg_val, value) \ (((reg_val) & ~CS_STATUS_WAIT_SB_SOURCE_MASK) | \ (((value) << CS_STATUS_WAIT_SB_SOURCE_SHIFT) & CS_STATUS_WAIT_SB_SOURCE_MASK)) #define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT 24 -#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK (0xF << CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) +#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK (0xFU << CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) #define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GET(reg_val) \ (((reg_val)&CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK) >> \ CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) @@ -630,28 +630,28 @@ /* End of CS_STATUS_WAIT_SYNC_WAIT_CONDITION values */ /* PROGRESS_WAIT is only for before v14.x.4 */ #define CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT 28 -#define CS_STATUS_WAIT_PROGRESS_WAIT_MASK (0x1 << CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) +#define CS_STATUS_WAIT_PROGRESS_WAIT_MASK (0x1U << CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) #define CS_STATUS_WAIT_PROGRESS_WAIT_GET(reg_val) \ (((reg_val)&CS_STATUS_WAIT_PROGRESS_WAIT_MASK) >> CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) #define CS_STATUS_WAIT_PROGRESS_WAIT_SET(reg_val, value) \ (((reg_val) & ~CS_STATUS_WAIT_PROGRESS_WAIT_MASK) | \ (((value) << CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) & CS_STATUS_WAIT_PROGRESS_WAIT_MASK)) #define CS_STATUS_WAIT_PROTM_PEND_SHIFT 29 -#define CS_STATUS_WAIT_PROTM_PEND_MASK (0x1 << CS_STATUS_WAIT_PROTM_PEND_SHIFT) +#define CS_STATUS_WAIT_PROTM_PEND_MASK (0x1U << CS_STATUS_WAIT_PROTM_PEND_SHIFT) #define CS_STATUS_WAIT_PROTM_PEND_GET(reg_val) \ (((reg_val)&CS_STATUS_WAIT_PROTM_PEND_MASK) >> CS_STATUS_WAIT_PROTM_PEND_SHIFT) #define CS_STATUS_WAIT_PROTM_PEND_SET(reg_val, value) \ (((reg_val) & ~CS_STATUS_WAIT_PROTM_PEND_MASK) | \ (((value) << CS_STATUS_WAIT_PROTM_PEND_SHIFT) & CS_STATUS_WAIT_PROTM_PEND_MASK)) #define CS_STATUS_WAIT_SYNC_WAIT_SIZE_SHIFT 30 -#define CS_STATUS_WAIT_SYNC_WAIT_SIZE_MASK (0x1 << CS_STATUS_WAIT_SYNC_WAIT_SIZE_SHIFT) +#define CS_STATUS_WAIT_SYNC_WAIT_SIZE_MASK (0x1U << CS_STATUS_WAIT_SYNC_WAIT_SIZE_SHIFT) #define CS_STATUS_WAIT_SYNC_WAIT_SIZE_GET(reg_val) \ (((reg_val)&CS_STATUS_WAIT_SYNC_WAIT_SIZE_MASK) >> CS_STATUS_WAIT_SYNC_WAIT_SIZE_SHIFT) #define CS_STATUS_WAIT_SYNC_WAIT_SIZE_SET(reg_val, value) \ (((reg_val) & ~CS_STATUS_WAIT_SYNC_WAIT_SIZE_MASK) | \ (((value) << CS_STATUS_WAIT_SYNC_WAIT_SIZE_SHIFT) & CS_STATUS_WAIT_SYNC_WAIT_SIZE_MASK)) #define CS_STATUS_WAIT_SYNC_WAIT_SHIFT 31 -#define CS_STATUS_WAIT_SYNC_WAIT_MASK (0x1 << CS_STATUS_WAIT_SYNC_WAIT_SHIFT) +#define CS_STATUS_WAIT_SYNC_WAIT_MASK (0x1U << CS_STATUS_WAIT_SYNC_WAIT_SHIFT) #define CS_STATUS_WAIT_SYNC_WAIT_GET(reg_val) \ (((reg_val)&CS_STATUS_WAIT_SYNC_WAIT_MASK) >> CS_STATUS_WAIT_SYNC_WAIT_SHIFT) #define CS_STATUS_WAIT_SYNC_WAIT_SET(reg_val, value) \ @@ -754,7 +754,7 @@ /* CS_FAULT register */ #define CS_FAULT_EXCEPTION_TYPE_SHIFT 0 -#define CS_FAULT_EXCEPTION_TYPE_MASK (0xFF << CS_FAULT_EXCEPTION_TYPE_SHIFT) +#define CS_FAULT_EXCEPTION_TYPE_MASK (0xFFU << CS_FAULT_EXCEPTION_TYPE_SHIFT) #define CS_FAULT_EXCEPTION_TYPE_GET(reg_val) \ (((reg_val)&CS_FAULT_EXCEPTION_TYPE_MASK) >> CS_FAULT_EXCEPTION_TYPE_SHIFT) #define CS_FAULT_EXCEPTION_TYPE_SET(reg_val, value) \ @@ -796,7 +796,7 @@ #define CS_FAULT_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_3 0xEB /* End of CS_FAULT_EXCEPTION_TYPE values */ #define CS_FAULT_EXCEPTION_DATA_SHIFT 8 -#define CS_FAULT_EXCEPTION_DATA_MASK (0xFFFFFF << CS_FAULT_EXCEPTION_DATA_SHIFT) +#define CS_FAULT_EXCEPTION_DATA_MASK (0xFFFFFFU << CS_FAULT_EXCEPTION_DATA_SHIFT) #define CS_FAULT_EXCEPTION_DATA_GET(reg_val) \ (((reg_val)&CS_FAULT_EXCEPTION_DATA_MASK) >> CS_FAULT_EXCEPTION_DATA_SHIFT) #define CS_FAULT_EXCEPTION_DATA_SET(reg_val, value) \ @@ -821,7 +821,7 @@ #define CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR 0x68 /* End of CS_FATAL_EXCEPTION_TYPE values */ #define CS_FATAL_EXCEPTION_DATA_SHIFT 8 -#define CS_FATAL_EXCEPTION_DATA_MASK (0xFFFFFF << CS_FATAL_EXCEPTION_DATA_SHIFT) +#define CS_FATAL_EXCEPTION_DATA_MASK (0xFFFFFFU << CS_FATAL_EXCEPTION_DATA_SHIFT) #define CS_FATAL_EXCEPTION_DATA_GET(reg_val) \ (((reg_val)&CS_FATAL_EXCEPTION_DATA_MASK) >> CS_FATAL_EXCEPTION_DATA_SHIFT) #define CS_FATAL_EXCEPTION_DATA_SET(reg_val, value) \ @@ -941,32 +941,32 @@ #define CSG_REQ_STATE_RESUME 0x3 /* End of CSG_REQ_STATE values */ #define CSG_REQ_EP_CFG_SHIFT 4 -#define CSG_REQ_EP_CFG_MASK (0x1 << CSG_REQ_EP_CFG_SHIFT) +#define CSG_REQ_EP_CFG_MASK (0x1U << CSG_REQ_EP_CFG_SHIFT) #define CSG_REQ_EP_CFG_GET(reg_val) (((reg_val)&CSG_REQ_EP_CFG_MASK) >> CSG_REQ_EP_CFG_SHIFT) #define CSG_REQ_EP_CFG_SET(reg_val, value) \ (((reg_val) & ~CSG_REQ_EP_CFG_MASK) | \ (((value) << CSG_REQ_EP_CFG_SHIFT) & CSG_REQ_EP_CFG_MASK)) #define CSG_REQ_STATUS_UPDATE_SHIFT 5 -#define CSG_REQ_STATUS_UPDATE_MASK (0x1 << CSG_REQ_STATUS_UPDATE_SHIFT) +#define CSG_REQ_STATUS_UPDATE_MASK (0x1U << CSG_REQ_STATUS_UPDATE_SHIFT) #define CSG_REQ_STATUS_UPDATE_GET(reg_val) \ (((reg_val)&CSG_REQ_STATUS_UPDATE_MASK) >> CSG_REQ_STATUS_UPDATE_SHIFT) #define CSG_REQ_STATUS_UPDATE_SET(reg_val, value) \ (((reg_val) & ~CSG_REQ_STATUS_UPDATE_MASK) | \ (((value) << CSG_REQ_STATUS_UPDATE_SHIFT) & CSG_REQ_STATUS_UPDATE_MASK)) #define CSG_REQ_SYNC_UPDATE_SHIFT 28 -#define CSG_REQ_SYNC_UPDATE_MASK (0x1 << CSG_REQ_SYNC_UPDATE_SHIFT) +#define CSG_REQ_SYNC_UPDATE_MASK (0x1U << CSG_REQ_SYNC_UPDATE_SHIFT) #define CSG_REQ_SYNC_UPDATE_GET(reg_val) \ (((reg_val)&CSG_REQ_SYNC_UPDATE_MASK) >> CSG_REQ_SYNC_UPDATE_SHIFT) #define CSG_REQ_SYNC_UPDATE_SET(reg_val, value) \ (((reg_val) & ~CSG_REQ_SYNC_UPDATE_MASK) | \ (((value) << CSG_REQ_SYNC_UPDATE_SHIFT) & CSG_REQ_SYNC_UPDATE_MASK)) #define CSG_REQ_IDLE_SHIFT 29 -#define CSG_REQ_IDLE_MASK (0x1 << CSG_REQ_IDLE_SHIFT) +#define CSG_REQ_IDLE_MASK (0x1U << CSG_REQ_IDLE_SHIFT) #define CSG_REQ_IDLE_GET(reg_val) (((reg_val)&CSG_REQ_IDLE_MASK) >> CSG_REQ_IDLE_SHIFT) #define CSG_REQ_IDLE_SET(reg_val, value) \ (((reg_val) & ~CSG_REQ_IDLE_MASK) | (((value) << CSG_REQ_IDLE_SHIFT) & CSG_REQ_IDLE_MASK)) #define CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT 31 -#define CSG_REQ_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) +#define CSG_REQ_PROGRESS_TIMER_EVENT_MASK (0x1U << CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) #define CSG_REQ_PROGRESS_TIMER_EVENT_GET(reg_val) \ (((reg_val)&CSG_REQ_PROGRESS_TIMER_EVENT_MASK) >> CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) #define CSG_REQ_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ @@ -1133,38 +1133,38 @@ #define CSG_ACK_STATE_RESUME 0x3 /* End of CSG_ACK_STATE values */ #define CSG_ACK_EP_CFG_SHIFT 4 -#define CSG_ACK_EP_CFG_MASK (0x1 << CSG_ACK_EP_CFG_SHIFT) +#define CSG_ACK_EP_CFG_MASK (0x1U << CSG_ACK_EP_CFG_SHIFT) #define CSG_ACK_EP_CFG_GET(reg_val) (((reg_val)&CSG_ACK_EP_CFG_MASK) >> CSG_ACK_EP_CFG_SHIFT) #define CSG_ACK_EP_CFG_SET(reg_val, value) \ (((reg_val) & ~CSG_ACK_EP_CFG_MASK) | \ (((value) << CSG_ACK_EP_CFG_SHIFT) & CSG_ACK_EP_CFG_MASK)) #define CSG_ACK_STATUS_UPDATE_SHIFT 5 -#define CSG_ACK_STATUS_UPDATE_MASK (0x1 << CSG_ACK_STATUS_UPDATE_SHIFT) +#define CSG_ACK_STATUS_UPDATE_MASK (0x1U << CSG_ACK_STATUS_UPDATE_SHIFT) #define CSG_ACK_STATUS_UPDATE_GET(reg_val) \ (((reg_val)&CSG_ACK_STATUS_UPDATE_MASK) >> CSG_ACK_STATUS_UPDATE_SHIFT) #define CSG_ACK_STATUS_UPDATE_SET(reg_val, value) \ (((reg_val) & ~CSG_ACK_STATUS_UPDATE_MASK) | \ (((value) << CSG_ACK_STATUS_UPDATE_SHIFT) & CSG_ACK_STATUS_UPDATE_MASK)) #define CSG_ACK_SYNC_UPDATE_SHIFT 28 -#define CSG_ACK_SYNC_UPDATE_MASK (0x1 << CSG_ACK_SYNC_UPDATE_SHIFT) +#define CSG_ACK_SYNC_UPDATE_MASK (0x1U << CSG_ACK_SYNC_UPDATE_SHIFT) #define CSG_ACK_SYNC_UPDATE_GET(reg_val) \ (((reg_val)&CSG_ACK_SYNC_UPDATE_MASK) >> CSG_ACK_SYNC_UPDATE_SHIFT) #define CSG_ACK_SYNC_UPDATE_SET(reg_val, value) \ (((reg_val) & ~CSG_ACK_SYNC_UPDATE_MASK) | \ (((value) << CSG_ACK_SYNC_UPDATE_SHIFT) & CSG_ACK_SYNC_UPDATE_MASK)) #define CSG_ACK_IDLE_SHIFT 29 -#define CSG_ACK_IDLE_MASK (0x1 << CSG_ACK_IDLE_SHIFT) +#define CSG_ACK_IDLE_MASK (0x1U << CSG_ACK_IDLE_SHIFT) #define CSG_ACK_IDLE_GET(reg_val) (((reg_val)&CSG_ACK_IDLE_MASK) >> CSG_ACK_IDLE_SHIFT) #define CSG_ACK_IDLE_SET(reg_val, value) \ (((reg_val) & ~CSG_ACK_IDLE_MASK) | (((value) << CSG_ACK_IDLE_SHIFT) & CSG_ACK_IDLE_MASK)) #define CSG_ACK_DOORBELL_SHIFT 30 -#define CSG_ACK_DOORBELL_MASK (0x1 << CSG_ACK_DOORBELL_SHIFT) +#define CSG_ACK_DOORBELL_MASK (0x1U << CSG_ACK_DOORBELL_SHIFT) #define CSG_ACK_DOORBELL_GET(reg_val) (((reg_val)&CSG_ACK_DOORBELL_MASK) >> CSG_ACK_DOORBELL_SHIFT) #define CSG_ACK_DOORBELL_SET(reg_val, value) \ (((reg_val) & ~CSG_ACK_DOORBELL_MASK) | \ (((value) << CSG_ACK_DOORBELL_SHIFT) & CSG_ACK_DOORBELL_MASK)) #define CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT 31 -#define CSG_ACK_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) +#define CSG_ACK_PROGRESS_TIMER_EVENT_MASK (0x1U << CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) #define CSG_ACK_PROGRESS_TIMER_EVENT_GET(reg_val) \ (((reg_val)&CSG_ACK_PROGRESS_TIMER_EVENT_MASK) >> CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) #define CSG_ACK_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ @@ -1611,8 +1611,8 @@ (((reg_val) & ~GLB_PWROFF_TIMER_TIMER_SOURCE_MASK) | \ (((value) << GLB_PWROFF_TIMER_TIMER_SOURCE_SHIFT) & GLB_PWROFF_TIMER_TIMER_SOURCE_MASK)) /* GLB_PWROFF_TIMER_TIMER_SOURCE values */ -#define GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP 0x0 -#define GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER 0x1 +#define GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP 0x0U +#define GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER 0x1U /* End of GLB_PWROFF_TIMER_TIMER_SOURCE values */ /* GLB_PWROFF_TIMER_CONFIG register */ @@ -1695,8 +1695,8 @@ (((reg_val) & ~GLB_IDLE_TIMER_TIMER_SOURCE_MASK) | \ (((value) << GLB_IDLE_TIMER_TIMER_SOURCE_SHIFT) & GLB_IDLE_TIMER_TIMER_SOURCE_MASK)) /* GLB_IDLE_TIMER_TIMER_SOURCE values */ -#define GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP 0x0 -#define GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER 0x1 +#define GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP 0x0U +#define GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER 0x1U /* End of GLB_IDLE_TIMER_TIMER_SOURCE values */ /* GLB_IDLE_TIMER_CONFIG values */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c index c18ed5b9f6cc..240397ebc16d 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c @@ -240,13 +240,17 @@ static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_MASK)), kbase_reg_read32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_MASK)), kbase_reg_read32(kbdev, MMU_CONTROL_ENUM(IRQ_MASK))); - dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(PWR_OVERRIDE0)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(PWR_OVERRIDE1))); - dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x TILER_CONFIG=0x%08x", - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(SHADER_CONFIG)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_MMU_CONFIG)), - kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(TILER_CONFIG))); + if (kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(14, 10, 0)) { + dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", + kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(PWR_OVERRIDE0)), + kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(PWR_OVERRIDE1))); + dev_err(kbdev->dev, + " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x TILER_CONFIG=0x%08x", + kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(SHADER_CONFIG)), + kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_MMU_CONFIG)), + kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(TILER_CONFIG))); + } + } /** diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c index a477ee666838..81ddeb667d06 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c @@ -528,7 +528,7 @@ static int wait_for_scheduler_to_exit_sleep(struct kbase_device *kbdev) /* Usually Scheduler would remain in sleeping state until the * auto-suspend timer expires and all active CSGs are suspended. */ - sleep_exit_wait_time = autosuspend_delay + kbdev->reset_timeout_ms; + sleep_exit_wait_time = (u32)autosuspend_delay + kbdev->reset_timeout_ms; remaining = kbase_csf_timeout_in_jiffies(sleep_exit_wait_time); @@ -680,7 +680,7 @@ static void unassign_user_doorbell_from_queue(struct kbase_device *kbdev, struct queue->doorbell_nr = KBASEP_USER_DB_NR_INVALID; /* After this the dummy page would be mapped in */ unmap_mapping_range(kbdev->csf.db_filp->f_inode->i_mapping, - queue->db_file_offset << PAGE_SHIFT, PAGE_SIZE, 1); + (loff_t)(queue->db_file_offset << PAGE_SHIFT), PAGE_SIZE, 1); } mutex_unlock(&kbdev->csf.reg_lock); @@ -714,7 +714,7 @@ static void assign_user_doorbell_to_queue(struct kbase_device *kbdev, /* After this the real Hw doorbell page would be mapped in */ unmap_mapping_range(kbdev->csf.db_filp->f_inode->i_mapping, - queue->db_file_offset << PAGE_SHIFT, PAGE_SIZE, 1); + (loff_t)(queue->db_file_offset << PAGE_SHIFT), PAGE_SIZE, 1); } mutex_unlock(&kbdev->csf.reg_lock); @@ -791,44 +791,56 @@ static void enqueue_gpu_idle_work(struct kbase_csf_scheduler *const scheduler) queue_work(scheduler->idle_wq, &scheduler->gpu_idle_work); } -void kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev) +bool kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; - int non_idle_offslot_grps; bool can_suspend_on_idle; + bool invoke_pm_state_machine = false; lockdep_assert_held(&kbdev->hwaccess_lock); lockdep_assert_held(&scheduler->interrupt_lock); - non_idle_offslot_grps = atomic_read(&scheduler->non_idle_offslot_grps); can_suspend_on_idle = kbase_pm_idle_groups_sched_suspendable(kbdev); KBASE_KTRACE_ADD(kbdev, SCHEDULER_GPU_IDLE_EVENT_CAN_SUSPEND, NULL, - ((u64)(u32)non_idle_offslot_grps) | (((u64)can_suspend_on_idle) << 32)); + (((u64)can_suspend_on_idle) << 32)); - if (!non_idle_offslot_grps) { - if (can_suspend_on_idle) { - /* fast_gpu_idle_handling is protected by the - * interrupt_lock, which would prevent this from being - * updated whilst gpu_idle_worker() is executing. + if (can_suspend_on_idle) { + /* fast_gpu_idle_handling is protected by the + * interrupt_lock, which would prevent this from being + * updated whilst gpu_idle_worker() is executing. + */ + scheduler->fast_gpu_idle_handling = (kbdev->csf.gpu_idle_hysteresis_ns == 0) || + !kbase_csf_scheduler_all_csgs_idle(kbdev); + + /* If GPU idle event occurred after the runtime suspend was aborted due to + * DB_MIRROR irq then it suggests that Userspace submission didn't make GPU + * non-idle. So the planned resumption of scheduling can be cancelled and + * MCU can be put back to sleep state to re-trigger the runtime suspend. + */ + if (unlikely(kbdev->pm.backend.exit_gpu_sleep_mode && + kbdev->pm.backend.runtime_suspend_abort_reason == + ABORT_REASON_DB_MIRROR_IRQ)) { + /* Cancel the planned resumption of scheduling */ + kbdev->pm.backend.exit_gpu_sleep_mode = false; + kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_NONE; + /* PM state machine can be invoked to put MCU back to the sleep + * state right away and thereby re-trigger the runtime suspend. */ - scheduler->fast_gpu_idle_handling = - (kbdev->csf.gpu_idle_hysteresis_ns == 0) || - !kbase_csf_scheduler_all_csgs_idle(kbdev); - - /* The GPU idle worker relies on update_on_slot_queues_offsets() to have - * finished. It's queued before to reduce the time it takes till execution - * but it'll eventually be blocked by the scheduler->interrupt_lock. - */ - enqueue_gpu_idle_work(scheduler); - - /* The extract offsets are unused in fast GPU idle handling */ - if (!scheduler->fast_gpu_idle_handling) - update_on_slot_queues_offsets(kbdev); + invoke_pm_state_machine = true; } - } else { - /* Invoke the scheduling tick to get the non-idle suspended groups loaded soon */ - kbase_csf_scheduler_invoke_tick(kbdev); + + /* The GPU idle worker relies on update_on_slot_queues_offsets() to have + * finished. It's queued before to reduce the time it takes till execution + * but it'll eventually be blocked by the scheduler->interrupt_lock. + */ + enqueue_gpu_idle_work(scheduler); } + + /* The extract offsets are unused in fast GPU idle handling */ + if (!scheduler->fast_gpu_idle_handling) + update_on_slot_queues_offsets(kbdev); + + return invoke_pm_state_machine; } u32 kbase_csf_scheduler_get_nr_active_csgs_locked(struct kbase_device *kbdev) @@ -837,8 +849,8 @@ u32 kbase_csf_scheduler_get_nr_active_csgs_locked(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); - nr_active_csgs = bitmap_weight(kbdev->csf.scheduler.csg_inuse_bitmap, - kbdev->csf.global_iface.group_num); + nr_active_csgs = (u32)bitmap_weight(kbdev->csf.scheduler.csg_inuse_bitmap, + kbdev->csf.global_iface.group_num); return nr_active_csgs; } @@ -938,13 +950,15 @@ static void update_idle_protm_group_state_to_runnable(struct kbase_queue_group * static bool scheduler_protm_wait_quit(struct kbase_device *kbdev) { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; - long wt = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); + const unsigned int fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); + long wt = kbase_csf_timeout_in_jiffies(fw_timeout_ms); long remaining; bool success = true; lockdep_assert_held(&scheduler->lock); - KBASE_KTRACE_ADD(kbdev, SCHEDULER_PROTM_WAIT_QUIT_START, NULL, jiffies_to_msecs(wt)); + KBASE_KTRACE_ADD(kbdev, SCHEDULER_PROTM_WAIT_QUIT_START, NULL, + jiffies_to_msecs((unsigned long)wt)); remaining = wait_event_timeout(kbdev->csf.event_wait, !kbase_csf_scheduler_protected_mode_in_use(kbdev), wt); @@ -954,12 +968,13 @@ static bool scheduler_protm_wait_quit(struct kbase_device *kbdev) struct kbase_context *kctx = group ? group->kctx : NULL; dev_warn(kbdev->dev, "[%llu] Timeout (%d ms), protm_quit wait skipped", - kbase_backend_get_cycle_cnt(kbdev), kbdev->csf.fw_timeout_ms); + kbase_backend_get_cycle_cnt(kbdev), fw_timeout_ms); schedule_actions_trigger_df(kbdev, kctx, DF_PROTECTED_MODE_EXIT_TIMEOUT); success = false; } - KBASE_KTRACE_ADD(kbdev, SCHEDULER_PROTM_WAIT_QUIT_END, NULL, jiffies_to_msecs(remaining)); + KBASE_KTRACE_ADD(kbdev, SCHEDULER_PROTM_WAIT_QUIT_END, NULL, + jiffies_to_msecs((unsigned long)remaining)); return success; } @@ -1084,6 +1099,8 @@ static int scheduler_pm_active_after_sleep(struct kbase_device *kbdev, unsigned if (!WARN_ON(prev_count == U32_MAX)) kbdev->csf.scheduler.pm_active_count++; + kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_NONE; + /* On 0 => 1, make a pm_ctx_active request */ if (!prev_count) { spin_unlock_irqrestore(&kbdev->hwaccess_lock, *flags); @@ -1279,8 +1296,9 @@ static void update_idle_suspended_group_state(struct kbase_queue_group *group) unsigned int n_slots = group->kctx->kbdev->csf.global_iface.group_num; spin_lock_irqsave(&scheduler->interrupt_lock, flags); - n_idle = bitmap_weight(scheduler->csg_slots_idle_mask, n_slots); - n_used = bitmap_weight(scheduler->csg_inuse_bitmap, n_slots); + n_idle = (unsigned int)bitmap_weight(scheduler->csg_slots_idle_mask, + n_slots); + n_used = (unsigned int)bitmap_weight(scheduler->csg_inuse_bitmap, n_slots); spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); if (n_idle || n_used < scheduler->num_csg_slots_for_tick || @@ -1292,7 +1310,7 @@ static void update_idle_suspended_group_state(struct kbase_queue_group *group) new_val = atomic_inc_return(&scheduler->non_idle_offslot_grps); KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_INC, group, - new_val); + (u64)new_val); } int kbase_csf_scheduler_group_get_slot_locked(struct kbase_queue_group *group) @@ -1363,7 +1381,7 @@ bool kbase_csf_scheduler_group_events_enabled(struct kbase_device *kbdev, } struct kbase_queue_group *kbase_csf_scheduler_get_group_on_slot(struct kbase_device *kbdev, - int slot) + u32 slot) { lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); @@ -1378,7 +1396,8 @@ static int halt_stream_sync(struct kbase_queue *queue) struct kbase_csf_cmd_stream_group_info *ginfo; struct kbase_csf_cmd_stream_info *stream; int csi_index = queue->csi_index; - long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); + const unsigned int fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); + long remaining = kbase_csf_timeout_in_jiffies(fw_timeout_ms); unsigned long flags; if (WARN_ON(!group) || WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) @@ -1399,8 +1418,8 @@ static int halt_stream_sync(struct kbase_queue *queue) dev_warn( kbdev->dev, "[%llu] Timeout (%d ms) waiting for queue to start on csi %d bound to group %d on slot %d", - kbase_backend_get_cycle_cnt(kbdev), kbdev->csf.fw_timeout_ms, - csi_index, group->handle, group->csg_nr); + kbase_backend_get_cycle_cnt(kbdev), fw_timeout_ms, csi_index, + group->handle, group->csg_nr); if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) kbase_reset_gpu(kbdev); @@ -1408,7 +1427,7 @@ static int halt_stream_sync(struct kbase_queue *queue) return -ETIMEDOUT; } - remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); + remaining = kbase_csf_timeout_in_jiffies(fw_timeout_ms); } spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); @@ -1430,8 +1449,8 @@ static int halt_stream_sync(struct kbase_queue *queue) dev_warn( kbdev->dev, "[%llu] Timeout (%d ms) waiting for queue to stop on csi %d bound to group %d on slot %d", - kbase_backend_get_cycle_cnt(kbdev), kbdev->csf.fw_timeout_ms, - queue->csi_index, group->handle, group->csg_nr); + kbase_backend_get_cycle_cnt(kbdev), fw_timeout_ms, queue->csi_index, + group->handle, group->csg_nr); /* TODO GPUCORE-25328: The CSG can't be terminated, the GPU * will be reset as a work-around. @@ -1487,6 +1506,7 @@ static int sched_halt_stream(struct kbase_queue *queue) int slot; int err = 0; const u32 group_schedule_timeout = kbase_get_timeout_ms(kbdev, CSF_CSG_SUSPEND_TIMEOUT); + const u32 fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); if (WARN_ON(!group)) return -EINVAL; @@ -1585,15 +1605,14 @@ retry: kbdev->csf.event_wait, (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output( stream, CS_ACK)) == CS_ACK_STATE_STOP), - kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms)); + kbase_csf_timeout_in_jiffies(fw_timeout_ms)); if (!remaining) { dev_warn( kbdev->dev, "[%llu] Timeout (%d ms) waiting for queue stop ack on csi %d bound to group %d on slot %d", - kbase_backend_get_cycle_cnt(kbdev), - kbdev->csf.fw_timeout_ms, queue->csi_index, - group->handle, group->csg_nr); + kbase_backend_get_cycle_cnt(kbdev), fw_timeout_ms, + queue->csi_index, group->handle, group->csg_nr); err = -ETIMEDOUT; @@ -1728,7 +1747,7 @@ static void program_cs_trace_cfg(struct kbase_csf_cmd_stream_info *stream, * queue's register_ex call. */ if (kbase_csf_scheduler_queue_has_trace(queue)) { - u32 cs_cfg = CS_INSTR_CONFIG_JASID_SET(queue->trace_cfg, queue->kctx->as_nr); + u32 cs_cfg = CS_INSTR_CONFIG_JASID_SET(queue->trace_cfg, (u32)queue->kctx->as_nr); kbase_csf_firmware_cs_input(stream, CS_INSTR_CONFIG, cs_cfg); kbase_csf_firmware_cs_input(stream, CS_INSTR_BUFFER_SIZE, queue->trace_buffer_size); @@ -1801,7 +1820,7 @@ static void program_cs(struct kbase_device *kbdev, struct kbase_queue *queue, kbase_csf_firmware_cs_input(stream, CS_USER_OUTPUT_HI, user_output >> 32); kbase_csf_firmware_cs_input(stream, CS_CONFIG, - (queue->doorbell_nr << 8) | (queue->priority & 0xF)); + ((u32)queue->doorbell_nr << 8) | (queue->priority & 0xF)); /* Program the queue's cs_trace configuration */ program_cs_trace_cfg(stream, queue); @@ -2039,6 +2058,7 @@ static void halt_csg_slot(struct kbase_queue_group *group, bool suspend) struct kbase_device *kbdev = group->kctx->kbdev; struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; struct kbase_csf_csg_slot *csg_slot = kbdev->csf.scheduler.csg_slots; + const unsigned int fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); s8 slot; lockdep_assert_held(&kbdev->csf.scheduler.lock); @@ -2050,15 +2070,14 @@ static void halt_csg_slot(struct kbase_queue_group *group, bool suspend) /* When in transition, wait for it to complete */ if (atomic_read(&csg_slot[slot].state) == CSG_SLOT_READY2RUN) { - long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); + long remaining = kbase_csf_timeout_in_jiffies(fw_timeout_ms); dev_dbg(kbdev->dev, "slot %d wait for up-running\n", slot); remaining = wait_event_timeout(kbdev->csf.event_wait, csg_slot_running(kbdev, slot), remaining); if (!remaining) dev_warn(kbdev->dev, "[%llu] slot %d timeout (%d ms) on up-running\n", - kbase_backend_get_cycle_cnt(kbdev), slot, - kbdev->csf.fw_timeout_ms); + kbase_backend_get_cycle_cnt(kbdev), slot, fw_timeout_ms); } if (csg_slot_running(kbdev, slot)) { @@ -2079,7 +2098,7 @@ static void halt_csg_slot(struct kbase_queue_group *group, bool suspend) csg_slot[slot].trigger_jiffies = jiffies; KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOP_REQ, group, halt_cmd); - KBASE_TLSTREAM_TL_KBASE_DEVICE_HALTING_CSG(kbdev, kbdev->id, slot, suspend); + KBASE_TLSTREAM_TL_KBASE_DEVICE_HALTING_CSG(kbdev, kbdev->id, (u32)slot, suspend); } } @@ -2554,7 +2573,8 @@ static void update_offslot_non_idle_cnt(struct kbase_queue_group *group) if (group->prepared_seq_num < scheduler->non_idle_scanout_grps) { int new_val = atomic_dec_return(&scheduler->non_idle_offslot_grps); - KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_DEC, group, new_val); + KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_DEC, group, + (u64)new_val); } } @@ -2569,7 +2589,8 @@ static void update_offslot_non_idle_cnt_for_onslot_grp(struct kbase_queue_group if (group->prepared_seq_num < scheduler->non_idle_scanout_grps) { int new_val = atomic_dec_return(&scheduler->non_idle_offslot_grps); - KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_DEC, group, new_val); + KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_DEC, group, + (u64)new_val); } } @@ -2587,13 +2608,13 @@ static void update_offslot_non_idle_cnt_on_grp_suspend(struct kbase_queue_group if (group->run_state == KBASE_CSF_GROUP_SUSPENDED) { int new_val = atomic_inc_return(&scheduler->non_idle_offslot_grps); KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_INC, - group, new_val); + group, (u64)new_val); } } else { if (group->run_state != KBASE_CSF_GROUP_SUSPENDED) { int new_val = atomic_dec_return(&scheduler->non_idle_offslot_grps); KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_DEC, - group, new_val); + group, (u64)new_val); } } } else { @@ -2601,7 +2622,7 @@ static void update_offslot_non_idle_cnt_on_grp_suspend(struct kbase_queue_group if (group->run_state == KBASE_CSF_GROUP_SUSPENDED) { int new_val = atomic_inc_return(&scheduler->non_idle_offslot_grps); KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_INC, group, - new_val); + (u64)new_val); } } } @@ -2796,10 +2817,10 @@ static bool cleanup_csg_slot(struct kbase_queue_group *group) csg_slot->trigger_jiffies = jiffies; atomic_set(&csg_slot->state, CSG_SLOT_READY); - KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_CLEANED, group, slot); + KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_CLEANED, group, (u64)slot); dev_dbg(kbdev->dev, "Cleanup done for group %d on slot %d\n", group->handle, slot); - KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG(kbdev, kbdev->id, slot); + KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG(kbdev, kbdev->id, (u32)slot); /* Notify the group is off-slot and the csg_reg might be available for * resue with other groups in a 'lazy unbinding' style. @@ -2965,7 +2986,7 @@ static void program_csg_slot(struct kbase_queue_group *group, s8 slot, u8 prio) kbase_csf_firmware_csg_input(ginfo, CSG_EP_REQ_LO, ep_cfg & U32_MAX); /* Program the address space number assigned to the context */ - kbase_csf_firmware_csg_input(ginfo, CSG_CONFIG, kctx->as_nr); + kbase_csf_firmware_csg_input(ginfo, CSG_CONFIG, (u32)kctx->as_nr); kbase_csf_firmware_csg_input(ginfo, CSG_SUSPEND_BUF_LO, normal_suspend_buf & U32_MAX); kbase_csf_firmware_csg_input(ginfo, CSG_SUSPEND_BUF_HI, normal_suspend_buf >> 32); @@ -3012,7 +3033,8 @@ static void program_csg_slot(struct kbase_queue_group *group, s8 slot, u8 prio) /* Trace the programming of the CSG on the slot */ KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG(kbdev, kbdev->id, group->kctx->id, group->handle, - slot, (state == CSG_REQ_STATE_RESUME) ? 1 : 0); + (u32)slot, + (state == CSG_REQ_STATE_RESUME) ? 1 : 0); dev_dbg(kbdev->dev, "Starting group %d of context %d_%d on slot %d with priority %u\n", group->handle, kctx->tgid, kctx->id, slot, prio); @@ -3063,7 +3085,7 @@ static void sched_evict_group(struct kbase_queue_group *group, bool fault, group->run_state == KBASE_CSF_GROUP_RUNNABLE)) { int new_val = atomic_dec_return(&scheduler->non_idle_offslot_grps); KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_DEC, group, - new_val); + (u64)new_val); } for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { @@ -3109,7 +3131,8 @@ static void sched_evict_group(struct kbase_queue_group *group, bool fault, static int term_group_sync(struct kbase_queue_group *group) { struct kbase_device *kbdev = group->kctx->kbdev; - long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); + const unsigned int fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); + long remaining = kbase_csf_timeout_in_jiffies(fw_timeout_ms); int err = 0; term_csg_slot(group); @@ -3125,7 +3148,7 @@ static int term_group_sync(struct kbase_queue_group *group) dev_warn( kbdev->dev, "[%llu] term request timeout (%d ms) for group %d of context %d_%d on slot %d", - kbase_backend_get_cycle_cnt(kbdev), kbdev->csf.fw_timeout_ms, group->handle, + kbase_backend_get_cycle_cnt(kbdev), fw_timeout_ms, group->handle, group->kctx->tgid, group->kctx->id, group->csg_nr); if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) error_type = DF_PING_REQUEST_TIMEOUT; @@ -3276,7 +3299,8 @@ static int scheduler_group_schedule(struct kbase_queue_group *group) insert_group_to_runnable(&kbdev->csf.scheduler, group, KBASE_CSF_GROUP_RUNNABLE); /* A new group into the scheduler */ new_val = atomic_inc_return(&kbdev->csf.scheduler.non_idle_offslot_grps); - KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_INC, group, new_val); + KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_INC, group, + (u64)new_val); } /* Since a group has become active now, check if GPU needs to be @@ -3302,7 +3326,8 @@ static inline void set_max_csg_slots(struct kbase_device *kbdev) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; unsigned int total_csg_slots = kbdev->csf.global_iface.group_num; - unsigned int max_address_space_slots = kbdev->nr_hw_address_spaces - NUM_RESERVED_AS_SLOTS; + unsigned int max_address_space_slots = + (unsigned int)kbdev->nr_hw_address_spaces - NUM_RESERVED_AS_SLOTS; WARN_ON(scheduler->num_active_address_spaces > total_csg_slots); @@ -3325,7 +3350,8 @@ static inline void count_active_address_space(struct kbase_device *kbdev, { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; unsigned int total_csg_slots = kbdev->csf.global_iface.group_num; - unsigned int max_address_space_slots = kbdev->nr_hw_address_spaces - NUM_RESERVED_AS_SLOTS; + unsigned int max_address_space_slots = + (unsigned int)kbdev->nr_hw_address_spaces - NUM_RESERVED_AS_SLOTS; if (scheduler->ngrp_to_schedule <= total_csg_slots) { if (kctx->csf.sched.ngrp_to_schedule == 1) @@ -3358,9 +3384,9 @@ static inline void count_active_address_space(struct kbase_device *kbdev, * priority of a slot belonging to a higher priority idle group will always be * greater than the priority of a slot belonging to a lower priority non-idle * group, reflecting the original position of a group in the scan order (i.e - * static priority) 'scan_seq_num', which is set during the prepare phase of a - * tick/tock before the group is moved to 'idle_groups_to_schedule' list if it - * is idle. + * static priority) 'sched_act_seq_num', which is set during the prepare phase + * of a tick/tock before the group is moved to 'idle_groups_to_schedule' list if + * it is idle. * The priority range [MAX_CSG_SLOT_PRIORITY, 0] is partitioned with the first * 'slots_for_tick' groups in the original scan order are assigned a priority in * the subrange [MAX_CSG_SLOT_PRIORITY, MAX_CSG_SLOT_PRIORITY - slots_for_tick), @@ -3385,8 +3411,8 @@ static u8 get_slot_priority(struct kbase_queue_group *group) slot_prio = (u8)(MAX_CSG_SLOT_PRIORITY - used_slots); } else { /* There will be a mix of idle and non-idle groups. */ - if (group->scan_seq_num < slots_for_tick) - slot_prio = (u8)(MAX_CSG_SLOT_PRIORITY - group->scan_seq_num); + if (group->sched_act_seq_num < slots_for_tick) + slot_prio = (u8)(MAX_CSG_SLOT_PRIORITY - group->sched_act_seq_num); else if (MAX_CSG_SLOT_PRIORITY > (slots_for_tick + used_slots)) slot_prio = (u8)(MAX_CSG_SLOT_PRIORITY - (slots_for_tick + used_slots)); else @@ -3644,7 +3670,7 @@ static void program_suspending_csg_slots(struct kbase_device *kbdev) "[%llu] Group %d of context %d_%d on slot %u failed to suspend (timeout %d ms)", kbase_backend_get_cycle_cnt(kbdev), group->handle, group->kctx->tgid, group->kctx->id, i, - kbdev->csf.fw_timeout_ms); + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) error_type = DF_PING_REQUEST_TIMEOUT; @@ -3713,7 +3739,8 @@ static void wait_csg_slots_start(struct kbase_device *kbdev) { u32 num_groups = kbdev->csf.global_iface.group_num; struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; - long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); + long remaining = + kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = { 0 }; u32 i; @@ -3753,8 +3780,9 @@ static void wait_csg_slots_start(struct kbase_device *kbdev) dev_err(kbdev->dev, "[%llu] Timeout (%d ms) waiting for CSG slots to start, slots: 0x%*pb\n", - kbase_backend_get_cycle_cnt(kbdev), kbdev->csf.fw_timeout_ms, - num_groups, slot_mask); + kbase_backend_get_cycle_cnt(kbdev), + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT), num_groups, + slot_mask); if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) error_type = DF_PING_REQUEST_TIMEOUT; schedule_actions_trigger_df(kbdev, group->kctx, error_type); @@ -3884,7 +3912,8 @@ static int wait_csg_slots_handshake_ack(struct kbase_device *kbdev, u32 field_ma static void wait_csg_slots_finish_prio_update(struct kbase_device *kbdev) { unsigned long *slot_mask = kbdev->csf.scheduler.csg_slots_prio_update; - long wait_time = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); + const unsigned int fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); + long wait_time = kbase_csf_timeout_in_jiffies(fw_timeout_ms); int ret = wait_csg_slots_handshake_ack(kbdev, CSG_REQ_EP_CFG_MASK, slot_mask, wait_time); lockdep_assert_held(&kbdev->csf.scheduler.lock); @@ -3898,7 +3927,7 @@ static void wait_csg_slots_finish_prio_update(struct kbase_device *kbdev) dev_warn( kbdev->dev, "[%llu] Timeout (%d ms) on CSG_REQ:EP_CFG, skipping the update wait: slot mask=0x%lx", - kbase_backend_get_cycle_cnt(kbdev), kbdev->csf.fw_timeout_ms, slot_mask[0]); + kbase_backend_get_cycle_cnt(kbdev), fw_timeout_ms, slot_mask[0]); if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) error_type = DF_PING_REQUEST_TIMEOUT; schedule_actions_trigger_df(kbdev, group->kctx, error_type); @@ -4297,6 +4326,34 @@ static void scheduler_apply(struct kbase_device *kbdev) program_suspending_csg_slots(kbdev); } +/** + * scheduler_scan_add_group_to_sched_list - Add a schedulable group to one of the scheduler's + * schedulable lists following exactly the scan sequence. + * + * @kbdev: Pointer to the GPU device. + * @sched_list: the scheduling list for appending the given group + * @group: the group that is to be added forllowing the scan out procedure. + * + * This function is called when a group is required to be added to a scheduler maintained list + * for a tick/tock scheduling operation. + */ +static void scheduler_scan_add_group_to_sched_list(struct kbase_device *kbdev, + struct list_head *sched_list, + struct kbase_queue_group *group) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + + lockdep_assert_held(&scheduler->lock); + lockdep_assert_held(&scheduler->interrupt_lock); + + list_add_tail(&group->link_to_schedule, sched_list); + /* Set the act_seq_num when a group is added to a schedulable list. + * Otherwise it's a don't care case, as the group would not be involved + * with onslot running. + */ + group->sched_act_seq_num = scheduler->csg_scan_sched_count++; +} + static void scheduler_ctx_scan_groups(struct kbase_device *kbdev, struct kbase_context *kctx, int priority, struct list_head *privileged_groups, struct list_head *active_groups) @@ -4338,17 +4395,17 @@ static void scheduler_ctx_scan_groups(struct kbase_device *kbdev, struct kbase_c update_idle_protm_group_state_to_runnable(group); else if (queue_group_idle_locked(group)) { if (can_schedule_idle_group(group)) - list_add_tail(&group->link_to_schedule, - &scheduler->idle_groups_to_schedule); + scheduler_scan_add_group_to_sched_list( + kbdev, &scheduler->idle_groups_to_schedule, group); continue; } if (protm_req && (group->priority == KBASE_QUEUE_GROUP_PRIORITY_REALTIME)) { - list_add_tail(&group->link_to_schedule, privileged_groups); + scheduler_scan_add_group_to_sched_list(kbdev, privileged_groups, group); continue; } - list_add_tail(&group->link_to_schedule, active_groups); + scheduler_scan_add_group_to_sched_list(kbdev, active_groups, group); } } @@ -4559,7 +4616,9 @@ static void scheduler_update_idle_slots_status(struct kbase_device *kbdev, /* The groups are aggregated into a single kernel doorbell request */ if (!bitmap_empty(csg_bitmap, num_groups)) { - long wt = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); + const unsigned int fw_timeout_ms = + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); + long wt = kbase_csf_timeout_in_jiffies(fw_timeout_ms); u32 db_slots = (u32)csg_bitmap[0]; kbase_csf_ring_csg_slots_doorbell(kbdev, db_slots); @@ -4574,8 +4633,7 @@ static void scheduler_update_idle_slots_status(struct kbase_device *kbdev, dev_warn( kbdev->dev, "[%llu] Timeout (%d ms) on CSG_REQ:STATUS_UPDATE, treat groups as not idle: slot mask=0x%lx", - kbase_backend_get_cycle_cnt(kbdev), kbdev->csf.fw_timeout_ms, - csg_bitmap[0]); + kbase_backend_get_cycle_cnt(kbdev), fw_timeout_ms, csg_bitmap[0]); schedule_actions_trigger_df(kbdev, group->kctx, DF_CSG_STATUS_UPDATE_TIMEOUT); @@ -4753,7 +4811,8 @@ static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, bool s dev_warn( kbdev->dev, "[%llu] Timeout (%d ms) waiting for CSG slots to suspend on power down, slot_mask: 0x%*pb\n", - kbase_backend_get_cycle_cnt(kbdev), kbdev->csf.fw_timeout_ms, + kbase_backend_get_cycle_cnt(kbdev), + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT), kbdev->csf.global_iface.group_num, slot_mask); if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) error_type = DF_PING_REQUEST_TIMEOUT; @@ -4843,42 +4902,29 @@ static bool scheduler_idle_suspendable(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); spin_lock(&scheduler->interrupt_lock); - if (scheduler->fast_gpu_idle_handling) { - scheduler->fast_gpu_idle_handling = false; - - if (scheduler->total_runnable_grps) { - suspend = !atomic_read(&scheduler->non_idle_offslot_grps) && - kbase_pm_idle_groups_sched_suspendable(kbdev); - } else - suspend = kbase_pm_no_runnables_sched_suspendable(kbdev); - - if (suspend && unlikely(atomic_read(&scheduler->gpu_no_longer_idle))) - suspend = false; - - spin_unlock(&scheduler->interrupt_lock); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return suspend; - } - if (scheduler->total_runnable_grps) { /* Check both on-slots and off-slots groups idle status */ - suspend = kbase_csf_scheduler_all_csgs_idle(kbdev) && + suspend = (scheduler->fast_gpu_idle_handling || + kbase_csf_scheduler_all_csgs_idle(kbdev)) && !atomic_read(&scheduler->non_idle_offslot_grps) && kbase_pm_idle_groups_sched_suspendable(kbdev); } else suspend = kbase_pm_no_runnables_sched_suspendable(kbdev); + if (suspend && unlikely(atomic_read(&scheduler->gpu_no_longer_idle))) + suspend = false; + /* Confirm that all groups are actually idle before proceeding with * suspension as groups might potentially become active again without * informing the scheduler in case userspace rings a doorbell directly. */ - if (suspend && (unlikely(atomic_read(&scheduler->gpu_no_longer_idle)) || - unlikely(!all_on_slot_groups_remained_idle(kbdev)))) { + if (suspend && !scheduler->fast_gpu_idle_handling && + unlikely(!all_on_slot_groups_remained_idle(kbdev))) { dev_dbg(kbdev->dev, "GPU suspension skipped due to active CSGs"); suspend = false; } + scheduler->fast_gpu_idle_handling = false; spin_unlock(&scheduler->interrupt_lock); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -5021,6 +5067,7 @@ static int scheduler_prepare(struct kbase_device *kbdev) WARN_ON(!list_empty(&scheduler->idle_groups_to_schedule)); scheduler->num_active_address_spaces = 0; scheduler->num_csg_slots_for_tick = 0; + scheduler->csg_scan_sched_count = 0; bitmap_zero(scheduler->csg_slots_prio_update, MAX_SUPPORTED_CSGS); INIT_LIST_HEAD(&privileged_groups); INIT_LIST_HEAD(&active_groups); @@ -5031,9 +5078,19 @@ static int scheduler_prepare(struct kbase_device *kbdev) for (i = 0; i < KBASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { struct kbase_context *kctx; - list_for_each_entry(kctx, &scheduler->runnable_kctxs, csf.link) - scheduler_ctx_scan_groups(kbdev, kctx, i, &privileged_groups, - &active_groups); + /* Scan the per-priority list of groups twice, firstly for the + * prioritised contexts, then the normal ones. + */ + list_for_each_entry(kctx, &scheduler->runnable_kctxs, csf.link) { + if (atomic_read(&kctx->prioritized)) + scheduler_ctx_scan_groups(kbdev, kctx, i, &privileged_groups, + &active_groups); + } + list_for_each_entry(kctx, &scheduler->runnable_kctxs, csf.link) { + if (!atomic_read(&kctx->prioritized)) + scheduler_ctx_scan_groups(kbdev, kctx, i, &privileged_groups, + &active_groups); + } } spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); @@ -5051,7 +5108,7 @@ static int scheduler_prepare(struct kbase_device *kbdev) * of the tick. It will be subject to up/downs during the scheduler * active phase. */ - atomic_set(&scheduler->non_idle_offslot_grps, scheduler->non_idle_scanout_grps); + atomic_set(&scheduler->non_idle_offslot_grps, (int)scheduler->non_idle_scanout_grps); KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_INC, NULL, scheduler->non_idle_scanout_grps); @@ -5296,7 +5353,7 @@ static void evict_lru_or_blocked_csg(struct kbase_device *kbdev) } if (lru_idle_group != NULL) { - unsigned long slot_mask = 1 << lru_idle_group->csg_nr; + unsigned long slot_mask = 1UL << lru_idle_group->csg_nr; dev_dbg(kbdev->dev, "Suspending LRU idle group %d of context %d_%d on slot %d", lru_idle_group->handle, lru_idle_group->kctx->tgid, @@ -5310,7 +5367,8 @@ static void evict_lru_or_blocked_csg(struct kbase_device *kbdev) "[%llu] LRU idle group %d of context %d_%d failed to suspend on slot %d (timeout %d ms)", kbase_backend_get_cycle_cnt(kbdev), lru_idle_group->handle, lru_idle_group->kctx->tgid, lru_idle_group->kctx->id, - lru_idle_group->csg_nr, kbdev->csf.fw_timeout_ms); + lru_idle_group->csg_nr, + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) error_type = DF_PING_REQUEST_TIMEOUT; schedule_actions_trigger_df(kbdev, lru_idle_group->kctx, error_type); @@ -5323,7 +5381,6 @@ static void schedule_actions(struct kbase_device *kbdev, bool is_tick) struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; unsigned long flags; struct kbase_queue_group *protm_grp; - int ret; bool skip_scheduling_actions; bool skip_idle_slots_update; bool new_protm_top_grp = false; @@ -5332,8 +5389,7 @@ static void schedule_actions(struct kbase_device *kbdev, bool is_tick) kbase_reset_gpu_assert_prevented(kbdev); lockdep_assert_held(&scheduler->lock); - ret = kbase_csf_scheduler_wait_mcu_active(kbdev); - if (ret) { + if (kbase_csf_scheduler_wait_mcu_active(kbdev)) { dev_err(kbdev->dev, "Wait for MCU power on failed on scheduling tick/tock"); return; } @@ -5658,7 +5714,8 @@ static int suspend_active_queue_groups_on_reset(struct kbase_device *kbdev) * case. */ kbase_gpu_start_cache_clean(kbdev, GPU_COMMAND_CACHE_CLN_INV_L2_LSC); - ret2 = kbase_gpu_wait_cache_clean_timeout(kbdev, kbdev->mmu_or_gpu_cache_op_wait_time_ms); + ret2 = kbase_gpu_wait_cache_clean_timeout( + kbdev, kbase_get_timeout_ms(kbdev, MMU_AS_INACTIVE_WAIT_TIMEOUT)); if (ret2) { dev_err(kbdev->dev, "[%llu] Timeout waiting for CACHE_CLN_INV_L2_LSC", kbase_backend_get_cycle_cnt(kbdev)); @@ -5755,7 +5812,8 @@ static bool scheduler_handle_reset_in_protected_mode(struct kbase_device *kbdev) * anyways. */ new_val = atomic_inc_return(&scheduler->non_idle_offslot_grps); - KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_INC, group, new_val); + KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_GRP_INC, group, + (u64)new_val); } unlock: @@ -5898,9 +5956,13 @@ static void firmware_aliveness_monitor(struct work_struct *work) goto exit; } - kbase_csf_scheduler_wait_mcu_active(kbdev); + if (kbase_csf_scheduler_wait_mcu_active(kbdev)) { + dev_err(kbdev->dev, "Wait for MCU power on failed at fw aliveness monitor"); + goto exit; + } - err = kbase_csf_firmware_ping_wait(kbdev, kbdev->csf.fw_timeout_ms); + err = kbase_csf_firmware_ping_wait(kbdev, + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); if (err) { /* It is acceptable to enqueue a reset whilst we've prevented @@ -6414,7 +6476,10 @@ static void check_group_sync_update_worker(struct work_struct *work) #if IS_ENABLED(CONFIG_DEBUG_FS) if (unlikely(scheduler->state == SCHED_BUSY)) { - queue_work(kctx->csf.sched.sync_update_wq, &kctx->csf.sched.sync_update_work); + queue_work(atomic_read(&kctx->prioritized) ? + kctx->csf.sched.sync_update_wq_high_prio : + kctx->csf.sched.sync_update_wq_normal_prio, + &kctx->csf.sched.sync_update_work); mutex_unlock(&scheduler->lock); return; } @@ -6457,10 +6522,13 @@ static void check_group_sync_update_worker(struct work_struct *work) static enum kbase_csf_event_callback_action check_group_sync_update_cb(void *param) { struct kbase_context *const kctx = param; + struct workqueue_struct *wq = atomic_read(&kctx->prioritized) ? + kctx->csf.sched.sync_update_wq_high_prio : + kctx->csf.sched.sync_update_wq_normal_prio; KBASE_KTRACE_ADD(kctx->kbdev, SCHEDULER_GROUP_SYNC_UPDATE_EVENT, kctx, 0u); - queue_work(kctx->csf.sched.sync_update_wq, &kctx->csf.sched.sync_update_work); + queue_work(wq, &kctx->csf.sched.sync_update_work); return KBASE_CSF_EVENT_CALLBACK_KEEP; } @@ -6483,12 +6551,22 @@ int kbase_csf_scheduler_context_init(struct kbase_context *kctx) INIT_LIST_HEAD(&kctx->csf.sched.idle_wait_groups); - kctx->csf.sched.sync_update_wq = - alloc_ordered_workqueue("mali_kbase_csf_sync_update_wq", WQ_HIGHPRI); - if (!kctx->csf.sched.sync_update_wq) { - dev_err(kbdev->dev, "Failed to initialize scheduler context workqueue"); + kctx->csf.sched.sync_update_wq_high_prio = alloc_ordered_workqueue( + "mali_sync_wq_%i_high_prio", WQ_UNBOUND | WQ_HIGHPRI, kctx->tgid); + if (kctx->csf.sched.sync_update_wq_high_prio == NULL) { + dev_err(kbdev->dev, + "Failed to initialize scheduler context high-priority workqueue"); err = -ENOMEM; - goto alloc_wq_failed; + goto alloc_high_prio_wq_failed; + } + + kctx->csf.sched.sync_update_wq_normal_prio = + alloc_ordered_workqueue("mali_sync_wq_%i_normal_prio", 0, kctx->tgid); + if (kctx->csf.sched.sync_update_wq_normal_prio == NULL) { + dev_err(kbdev->dev, + "Failed to initialize scheduler context normal-priority workqueue"); + err = -ENOMEM; + goto alloc_normal_prio_wq_failed; } INIT_WORK(&kctx->csf.sched.sync_update_work, check_group_sync_update_worker); @@ -6505,8 +6583,10 @@ int kbase_csf_scheduler_context_init(struct kbase_context *kctx) return err; event_wait_add_failed: - destroy_workqueue(kctx->csf.sched.sync_update_wq); -alloc_wq_failed: + destroy_workqueue(kctx->csf.sched.sync_update_wq_normal_prio); +alloc_normal_prio_wq_failed: + destroy_workqueue(kctx->csf.sched.sync_update_wq_high_prio); +alloc_high_prio_wq_failed: kbase_ctx_sched_remove_ctx(kctx); #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) gpu_metrics_ctx_term(kctx); @@ -6518,7 +6598,8 @@ void kbase_csf_scheduler_context_term(struct kbase_context *kctx) { kbase_csf_event_wait_remove(kctx, check_group_sync_update_cb, kctx); cancel_work_sync(&kctx->csf.sched.sync_update_work); - destroy_workqueue(kctx->csf.sched.sync_update_wq); + destroy_workqueue(kctx->csf.sched.sync_update_wq_normal_prio); + destroy_workqueue(kctx->csf.sched.sync_update_wq_high_prio); kbase_ctx_sched_remove_ctx(kctx); #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) @@ -6995,6 +7076,7 @@ int kbase_csf_scheduler_handle_runtime_suspend(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->pm.backend.exit_gpu_sleep_mode = true; + kbdev->pm.backend.runtime_suspend_abort_reason = ABORT_REASON_NON_IDLE_CGS; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); kbase_csf_scheduler_invoke_tick(kbdev); @@ -7060,3 +7142,4 @@ void kbase_csf_scheduler_force_wakeup(struct kbase_device *kbdev) scheduler_wakeup(kbdev, true); mutex_unlock(&scheduler->lock); } +KBASE_EXPORT_TEST_API(kbase_csf_scheduler_force_wakeup); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h index abd62342d38f..5047092d6650 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h @@ -122,7 +122,7 @@ bool kbase_csf_scheduler_group_events_enabled(struct kbase_device *kbdev, * Note: Caller must hold the interrupt_lock. */ struct kbase_queue_group *kbase_csf_scheduler_get_group_on_slot(struct kbase_device *kbdev, - int slot); + u32 slot); /** * kbase_csf_scheduler_group_deschedule() - Deschedule a GPU command queue @@ -591,8 +591,11 @@ int kbase_csf_scheduler_handle_runtime_suspend(struct kbase_device *kbdev); * @kbdev: Pointer to the device * * This function is called when a GPU idle IRQ has been raised. + * + * Return: true if the PM state machine needs to be invoked after the processing + * of GPU idle irq, otherwise false. */ -void kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev); +bool kbase_csf_scheduler_process_gpu_idle_event(struct kbase_device *kbdev); /** * kbase_csf_scheduler_get_nr_active_csgs() - Get the number of active CSGs diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_sync.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_sync.c index b95e77ce4eaa..aa88b5f59d3b 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_sync.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_sync.c @@ -99,11 +99,7 @@ static void kbasep_csf_sync_print_kcpu_fence_wait_or_signal(char *buffer, int *l struct kbase_kcpu_command *cmd, const char *cmd_name) { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence = NULL; -#else struct dma_fence *fence = NULL; -#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ struct kbase_kcpu_command_fence_info *fence_info; struct kbase_sync_fence_info info; const char *timeline_name = NULL; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c index f898535004c5..2d148eea025e 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c @@ -225,7 +225,7 @@ static void remove_unlinked_chunk(struct kbase_context *kctx, * For "no user free count", we check that the count is 1 as it is a shrinkable region; * no other code part within kbase can take a reference to it. */ - WARN_ON(atomic_read(&chunk->region->no_user_free_count) > 1); + WARN_ON(atomic64_read(&chunk->region->no_user_free_count) > 1); kbase_va_region_no_user_free_dec(chunk->region); #if !defined(CONFIG_MALI_VECTOR_DUMP) chunk->region->flags &= ~KBASE_REG_DONT_NEED; @@ -308,7 +308,7 @@ static struct kbase_csf_tiler_heap_chunk *alloc_new_chunk(struct kbase_context * * It should be fine and not a security risk if we let the region leak till * region tracker termination in such a case. */ - if (unlikely(atomic_read(&chunk->region->no_user_free_count) > 1)) { + if (unlikely(atomic64_read(&chunk->region->no_user_free_count) > 1)) { dev_err(kctx->kbdev->dev, "Chunk region has no_user_free_count > 1!\n"); goto unroll_region; } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.c index 788cfb2ad601..a2bb49422e98 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap_reclaim.c @@ -29,7 +29,7 @@ #define HEAP_SHRINKER_SEEKS (DEFAULT_SEEKS + 2) /* Tiler heap shrinker batch value */ -#define HEAP_SHRINKER_BATCH (512) +#define HEAP_SHRINKER_BATCH (512 / (GPU_PAGES_PER_CPU_PAGE)) /* Tiler heap reclaim scan (free) method size for limiting a scan run length */ #define HEAP_RECLAIM_SCAN_BATCH_SIZE (HEAP_SHRINKER_BATCH << 7) @@ -69,8 +69,8 @@ static void detach_ctx_from_heap_reclaim_mgr(struct kbase_context *kctx) list_del_init(&info->mgr_link); if (remaining) - WARN_ON(atomic_sub_return(remaining, &scheduler->reclaim_mgr.unused_pages) < - 0); + WARN_ON(atomic_sub_return((int)remaining, + &scheduler->reclaim_mgr.unused_pages) < 0); dev_dbg(kctx->kbdev->dev, "Reclaim_mgr_detach: ctx_%d_%d, est_pages=0%u, freed_pages=%u", kctx->tgid, @@ -96,7 +96,7 @@ static void attach_ctx_to_heap_reclaim_mgr(struct kbase_context *kctx) list_add_tail(&info->mgr_link, &scheduler->reclaim_mgr.ctx_lists[prio]); /* Accumulate the estimated pages to the manager total field */ - atomic_add(info->nr_est_unused_pages, &scheduler->reclaim_mgr.unused_pages); + atomic_add((int)info->nr_est_unused_pages, &scheduler->reclaim_mgr.unused_pages); dev_dbg(kctx->kbdev->dev, "Reclaim_mgr_attach: ctx_%d_%d, est_count_pages=%u", kctx->tgid, kctx->id, info->nr_est_unused_pages); @@ -201,8 +201,8 @@ static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev) * headers of the individual chunks and buffer descriptors. */ kbase_gpu_start_cache_clean(kbdev, GPU_COMMAND_CACHE_CLN_INV_L2); - if (kbase_gpu_wait_cache_clean_timeout(kbdev, - kbdev->mmu_or_gpu_cache_op_wait_time_ms)) + if (kbase_gpu_wait_cache_clean_timeout( + kbdev, kbase_get_timeout_ms(kbdev, MMU_AS_INACTIVE_WAIT_TIMEOUT))) dev_warn( kbdev->dev, "[%llu] Timeout waiting for CACHE_CLN_INV_L2 to complete before Tiler heap reclaim", @@ -241,7 +241,7 @@ static unsigned long reclaim_unused_heap_pages(struct kbase_device *kbdev) u32 rm_cnt = MIN(info->nr_est_unused_pages - info->nr_freed_pages, freed_pages); - WARN_ON(atomic_sub_return(rm_cnt, &mgr->unused_pages) < 0); + WARN_ON(atomic_sub_return((int)rm_cnt, &mgr->unused_pages) < 0); /* tracking the freed pages, before a potential detach call */ info->nr_freed_pages += freed_pages; @@ -278,7 +278,7 @@ static unsigned long kbase_csf_tiler_heap_reclaim_count_free_pages(struct kbase_ struct shrink_control *sc) { struct kbase_csf_sched_heap_reclaim_mgr *mgr = &kbdev->csf.scheduler.reclaim_mgr; - unsigned long page_cnt = atomic_read(&mgr->unused_pages); + unsigned long page_cnt = (unsigned long)atomic_read(&mgr->unused_pages); CSTD_UNUSED(sc); @@ -300,14 +300,14 @@ static unsigned long kbase_csf_tiler_heap_reclaim_scan_free_pages(struct kbase_d /* Wait for roughly 2-ms */ wait_event_timeout(kbdev->csf.event_wait, (scheduler->state != SCHED_BUSY), - msecs_to_jiffies(2)); + (long)msecs_to_jiffies(2)); if (!mutex_trylock(&kbdev->csf.scheduler.lock)) { dev_dbg(kbdev->dev, "Tiler heap reclaim scan see device busy (freed: 0)"); return 0; } } - avail = atomic_read(&mgr->unused_pages); + avail = (unsigned long)atomic_read(&mgr->unused_pages); if (avail) freed = reclaim_unused_heap_pages(kbdev); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c index 3b2e5ae6ae79..7845ad1397d6 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_timeout.c @@ -51,7 +51,7 @@ static int set_timeout(struct kbase_device *const kbdev, u64 const timeout) dev_dbg(kbdev->dev, "New progress timeout: %llu cycles\n", timeout); - atomic64_set(&kbdev->csf.progress_timeout, timeout); + atomic64_set(&kbdev->csf.progress_timeout, (s64)timeout); kbase_device_set_timeout(kbdev, CSF_SCHED_PROTM_PROGRESS_TIMEOUT, timeout, 1); return 0; @@ -112,7 +112,7 @@ static ssize_t progress_timeout_store(struct device *const dev, struct device_at if (err) return err; - return count; + return (ssize_t)count; } /** @@ -179,5 +179,5 @@ void kbase_csf_timeout_term(struct kbase_device *const kbdev) u64 kbase_csf_timeout_get(struct kbase_device *const kbdev) { - return atomic64_read(&kbdev->csf.progress_timeout); + return (u64)atomic64_read(&kbdev->csf.progress_timeout); } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_debug_csf_fault.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_debug_csf_fault.c index 2e87c08ded0e..32c34b2e04d7 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_debug_csf_fault.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_debug_csf_fault.c @@ -88,8 +88,8 @@ bool kbase_debug_csf_fault_notify(struct kbase_device *kbdev, struct kbase_conte goto unlock; } - kbdev->csf.dof.kctx_tgid = kctx ? kctx->tgid : 0; - kbdev->csf.dof.kctx_id = kctx ? kctx->id : 0; + kbdev->csf.dof.kctx_tgid = kctx ? (unsigned int)kctx->tgid : 0U; + kbdev->csf.dof.kctx_id = kctx ? kctx->id : 0U; kbdev->csf.dof.error_code = error; kbase_debug_csf_fault_wakeup(kbdev); @@ -142,7 +142,7 @@ static ssize_t debug_csf_fault_read(struct file *file, char __user *buffer, size spin_unlock_irqrestore(&kbdev->csf.dof.lock, flags); dev_info(kbdev->dev, "debug csf fault info read"); - return simple_read_from_buffer(buffer, size, f_pos, buf, count); + return simple_read_from_buffer(buffer, size, f_pos, buf, (size_t)count); } static int debug_csf_fault_open(struct inode *in, struct file *file) @@ -197,7 +197,7 @@ static ssize_t debug_csf_fault_write(struct file *file, const char __user *ubuf, */ wake_up(&kbdev->csf.dof.dump_wait_wq); - return count; + return (ssize_t)count; } static int debug_csf_fault_release(struct inode *in, struct file *file) diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c index 46ad63692a34..fe8201f7f7e6 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c @@ -778,7 +778,8 @@ KBASE_EXPORT_TEST_API(kbase_debug_coresight_csf_state_check); bool kbase_debug_coresight_csf_state_wait(struct kbase_device *kbdev, enum kbase_debug_coresight_csf_state state) { - const long wait_timeout = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); + const long wait_timeout = + kbase_csf_timeout_in_jiffies(kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); struct kbase_debug_coresight_csf_config *config_entry, *next_config_entry; unsigned long flags; bool success = true; diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c index 46eb6db36c82..ec5ca10e135b 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_csf.c @@ -27,8 +27,9 @@ void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) { - *written += MAX( - snprintf(buffer + *written, MAX(sz - *written, 0), "group,slot,prio,csi,kcpu"), 0); + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + "group,slot,prio,csi,kcpu"), + 0); } void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char *buffer, int sz, @@ -43,38 +44,39 @@ void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char * if (be_msg->gpu.flags & KBASE_KTRACE_FLAG_CSF_GROUP) { const s8 slot = be_msg->gpu.csg_nr; /* group,slot, */ - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), "%u,%d,", + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%u,%d,", be_msg->gpu.group_handle, slot), 0); /* prio */ if (slot >= 0) - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), "%u", - be_msg->gpu.slot_prio), + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + "%u", be_msg->gpu.slot_prio), 0); /* , */ - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ","), 0); + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); } else { /* No group,slot,prio fields, but ensure ending with "," */ - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ",,,"), 0); + *written += + MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,,"), 0); } /* queue parts: csi */ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_CSF_QUEUE) - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), "%d", + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", be_msg->gpu.csi_index), 0); /* , */ - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ","), 0); + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); if (be_msg->gpu.flags & KBASE_KTRACE_FLAG_CSF_KCPU) { /* kcpu data */ - *written += - MAX(snprintf(buffer + *written, MAX(sz - *written, 0), "kcpu %d (0x%llx)", - be_msg->kcpu.id, be_msg->kcpu.extra_info_val), - 0); + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + "kcpu %d (0x%llx)", be_msg->kcpu.id, + be_msg->kcpu.extra_info_val), + 0); } /* Don't end with a trailing "," - this is a 'standalone' formatted diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h index c1f60dd16981..9978e4939d14 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h @@ -98,7 +98,7 @@ union kbase_ktrace_backend { * KBASE_KTRACE_FLAG_JM_ATOM */ u64 gpu_addr; - int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ + unsigned int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ /* Pack smaller members together */ kbase_ktrace_code_t code; kbase_ktrace_flag_t flags; diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c index 8f95ca67f4d6..beac074f2035 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c @@ -27,7 +27,7 @@ void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) { - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "katom,gpu_addr,jobslot,refcount"), 0); } @@ -37,8 +37,8 @@ void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char * { /* katom */ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_ATOM) - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), - "atom %d (ud: 0x%llx 0x%llx)", + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + "atom %u (ud: 0x%llx 0x%llx)", trace_msg->backend.gpu.atom_number, trace_msg->backend.gpu.atom_udata[0], trace_msg->backend.gpu.atom_udata[1]), @@ -46,31 +46,32 @@ void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, char * /* gpu_addr */ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_BACKEND) - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ",%.8llx,", - trace_msg->backend.gpu.gpu_addr), + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), + ",%.8llx,", trace_msg->backend.gpu.gpu_addr), 0); else - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ",,"), 0); + *written += + MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ",,"), 0); /* jobslot */ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), "%d", + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", trace_msg->backend.gpu.jobslot), 0); - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), ","), 0); + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), ","), 0); /* refcount */ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), "%d", + *written += MAX(snprintf(buffer + *written, (size_t)MAX(sz - *written, 0), "%d", trace_msg->backend.gpu.refcount), 0); } void kbasep_ktrace_add_jm(struct kbase_device *kbdev, enum kbase_ktrace_code code, struct kbase_context *kctx, const struct kbase_jd_atom *katom, - u64 gpu_addr, kbase_ktrace_flag_t flags, int refcount, int jobslot, - u64 info_val) + u64 gpu_addr, kbase_ktrace_flag_t flags, int refcount, + unsigned int jobslot, u64 info_val) { unsigned long irqflags; struct kbase_ktrace_msg *trace_msg; diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h index b91176deac26..e9c78e1d9fc6 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h @@ -42,8 +42,8 @@ */ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, enum kbase_ktrace_code code, struct kbase_context *kctx, const struct kbase_jd_atom *katom, - u64 gpu_addr, kbase_ktrace_flag_t flags, int refcount, int jobslot, - u64 info_val); + u64 gpu_addr, kbase_ktrace_flag_t flags, int refcount, + unsigned int jobslot, u64 info_val); #define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, refcount, jobslot, \ info_val) \ @@ -175,7 +175,7 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, enum kbase_ktrace_code cod do { \ /* capture values that could come from non-pure function calls */ \ u64 __gpu_addr = gpu_addr; \ - int __jobslot = jobslot; \ + unsigned int __jobslot = jobslot; \ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, 0); \ KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \ @@ -202,7 +202,7 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, enum kbase_ktrace_code cod do { \ /* capture values that could come from non-pure function calls */ \ u64 __gpu_addr = gpu_addr; \ - int __jobslot = jobslot; \ + unsigned int __jobslot = jobslot; \ u64 __info_val = info_val; \ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, __info_val); \ @@ -234,7 +234,7 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, enum kbase_ktrace_code cod KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, 0u); \ KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, \ - __refcount); \ + (unsigned int)__refcount); \ } while (0) /** @@ -265,7 +265,7 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, enum kbase_ktrace_code cod KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, \ __info_val); \ KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, \ - __refcount, __info_val); \ + (unsigned int)__refcount, __info_val); \ } while (0) /** diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h index fd62bae9e0a9..3e5b0ec89758 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h @@ -28,7 +28,7 @@ #define _KBASE_DEBUG_LINUX_KTRACE_JM_H_ DECLARE_EVENT_CLASS(mali_jm_slot_template, - TP_PROTO(struct kbase_context *kctx, int jobslot, u64 info_val), + TP_PROTO(struct kbase_context *kctx, unsigned int jobslot, u64 info_val), TP_ARGS(kctx, jobslot, info_val), TP_STRUCT__entry(__field(pid_t, kctx_tgid) __field(u32, kctx_id) __field(unsigned int, jobslot) __field(u64, info_val)), @@ -38,9 +38,9 @@ DECLARE_EVENT_CLASS(mali_jm_slot_template, TP_printk("kctx=%d_%u jobslot=%u info=0x%llx", __entry->kctx_tgid, __entry->kctx_id, __entry->jobslot, __entry->info_val)); -#define DEFINE_MALI_JM_SLOT_EVENT(name) \ - DEFINE_EVENT(mali_jm_slot_template, mali_##name, \ - TP_PROTO(struct kbase_context *kctx, int jobslot, u64 info_val), \ +#define DEFINE_MALI_JM_SLOT_EVENT(name) \ + DEFINE_EVENT(mali_jm_slot_template, mali_##name, \ + TP_PROTO(struct kbase_context *kctx, unsigned int jobslot, u64 info_val), \ TP_ARGS(kctx, jobslot, info_val)) DEFINE_MALI_JM_SLOT_EVENT(JM_RETURN_ATOM_TO_JS); DEFINE_MALI_JM_SLOT_EVENT(JM_MARK_FOR_RETURN_TO_JS); @@ -78,7 +78,7 @@ DEFINE_MALI_JM_SLOT_EVENT(JS_SLOT_PRIO_IS_BLOCKED); #undef DEFINE_MALI_JM_SLOT_EVENT DECLARE_EVENT_CLASS(mali_jm_refcount_template, - TP_PROTO(struct kbase_context *kctx, int refcount, u64 info_val), + TP_PROTO(struct kbase_context *kctx, unsigned int refcount, u64 info_val), TP_ARGS(kctx, refcount, info_val), TP_STRUCT__entry(__field(pid_t, kctx_tgid) __field(u32, kctx_id) __field(unsigned int, refcount) @@ -89,9 +89,9 @@ DECLARE_EVENT_CLASS(mali_jm_refcount_template, TP_printk("kctx=%d_%u refcount=%u info=0x%llx", __entry->kctx_tgid, __entry->kctx_id, __entry->refcount, __entry->info_val)); -#define DEFINE_MALI_JM_REFCOUNT_EVENT(name) \ - DEFINE_EVENT(mali_jm_refcount_template, mali_##name, \ - TP_PROTO(struct kbase_context *kctx, int refcount, u64 info_val), \ +#define DEFINE_MALI_JM_REFCOUNT_EVENT(name) \ + DEFINE_EVENT(mali_jm_refcount_template, mali_##name, \ + TP_PROTO(struct kbase_context *kctx, unsigned int refcount, u64 info_val), \ TP_ARGS(kctx, refcount, info_val)) DEFINE_MALI_JM_REFCOUNT_EVENT(JS_ADD_JOB); DEFINE_MALI_JM_REFCOUNT_EVENT(JS_REMOVE_JOB); diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c index 12a722765c5e..0842460bc08a 100644 --- a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c @@ -71,13 +71,13 @@ static const char *const kbasep_ktrace_code_string[] = { static void kbasep_ktrace_format_header(char *buffer, int sz, s32 written) { - written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), "secs,thread_id,cpu,code,kctx,"), 0); kbasep_ktrace_backend_format_header(buffer, sz, &written); - written += MAX(snprintf(buffer + written, MAX(sz - written, 0), + written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), ",info_val,ktrace_version=%u.%u", KBASE_KTRACE_VERSION_MAJOR, KBASE_KTRACE_VERSION_MINOR), 0); @@ -93,7 +93,7 @@ static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, char *b * * secs,thread_id,cpu,code, */ - written += MAX(snprintf(buffer + written, MAX(sz - written, 0), "%d.%.6d,%d,%d,%s,", + written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), "%d.%.6d,%d,%d,%s,", (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, @@ -102,12 +102,12 @@ static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, char *b /* kctx part: */ if (trace_msg->kctx_tgid) { - written += MAX(snprintf(buffer + written, MAX(sz - written, 0), "%d_%u", + written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), "%d_%u", trace_msg->kctx_tgid, trace_msg->kctx_id), 0); } /* Trailing comma */ - written += MAX(snprintf(buffer + written, MAX(sz - written, 0), ","), 0); + written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), ","), 0); /* Backend parts */ kbasep_ktrace_backend_format_msg(trace_msg, buffer, sz, &written); @@ -119,7 +119,7 @@ static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, char *b * Note that the last column is empty, it's simply to hold the ktrace * version in the header */ - written += MAX(snprintf(buffer + written, MAX(sz - written, 0), ",0x%.16llx", + written += MAX(snprintf(buffer + written, (size_t)MAX(sz - written, 0), ",0x%.16llx", (unsigned long long)trace_msg->info_val), 0); buffer[sz - 1] = 0; @@ -156,7 +156,7 @@ void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace, struct kbase_ktrace_msg { lockdep_assert_held(&ktrace->lock); - trace_msg->thread_id = task_pid_nr(current); + trace_msg->thread_id = (u32)task_pid_nr(current); trace_msg->cpu = task_cpu(current); ktime_get_real_ts64(&trace_msg->timestamp); diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c index 36778923f364..52aa63330afe 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c @@ -87,8 +87,8 @@ static int kbase_backend_late_init(struct kbase_device *kbdev) #ifdef CONFIG_MALI_BIFROST_DEBUG #if IS_ENABLED(CONFIG_MALI_REAL_HW) - if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { - dev_err(kbdev->dev, "Interrupt assignment check failed.\n"); + if (kbase_validate_interrupts(kbdev) != 0) { + dev_err(kbdev->dev, "Interrupt validation failed.\n"); err = -EINVAL; goto fail_interrupt_test; } @@ -278,7 +278,7 @@ static const struct kbase_device_init dev_init[] = { #if !IS_ENABLED(CONFIG_MALI_REAL_HW) { kbase_gpu_device_create, kbase_gpu_device_destroy, "Dummy model initialization failed" }, #else /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ - { assign_irqs, NULL, "IRQ search failed" }, + { kbase_get_irqs, NULL, "IRQ search failed" }, #endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ #if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) { registers_map, registers_unmap, "Register map failed" }, diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c index c7f34bc78137..ab9df01610ab 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c @@ -175,4 +175,5 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); } +KBASE_EXPORT_TEST_API(kbase_gpu_interrupt); diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c index d530010c096e..f971b3b939df 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c @@ -101,3 +101,4 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); } +KBASE_EXPORT_TEST_API(kbase_gpu_interrupt); diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c index 556e388e11bd..ab46f858a542 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c @@ -75,8 +75,8 @@ static int kbase_backend_late_init(struct kbase_device *kbdev) #ifdef CONFIG_MALI_BIFROST_DEBUG #if IS_ENABLED(CONFIG_MALI_REAL_HW) - if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { - dev_err(kbdev->dev, "Interrupt assignment check failed.\n"); + if (kbase_validate_interrupts(kbdev) != 0) { + dev_err(kbdev->dev, "Interrupt validation failed.\n"); err = -EINVAL; goto fail_interrupt_test; } @@ -216,7 +216,7 @@ static const struct kbase_device_init dev_init[] = { #if !IS_ENABLED(CONFIG_MALI_REAL_HW) { kbase_gpu_device_create, kbase_gpu_device_destroy, "Dummy model initialization failed" }, #else /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ - { assign_irqs, NULL, "IRQ search failed" }, + { kbase_get_irqs, NULL, "IRQ search failed" }, #endif /* !IS_ENABLED(CONFIG_MALI_REAL_HW) */ #if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) { registers_map, registers_unmap, "Register map failed" }, diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device.c b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c index 5cf85517853d..b191c758c62f 100644 --- a/drivers/gpu/arm/bifrost/device/mali_kbase_device.c +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c @@ -67,7 +67,7 @@ static DEFINE_MUTEX(kbase_dev_list_lock); static LIST_HEAD(kbase_dev_list); -static int kbase_dev_nr; +static unsigned int kbase_dev_nr; struct kbase_device *kbase_device_alloc(void) { @@ -83,9 +83,10 @@ struct kbase_device *kbase_device_alloc(void) */ static int kbase_device_all_as_init(struct kbase_device *kbdev) { - int i, err = 0; + unsigned int i; + int err = 0; - for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { + for (i = 0; i < (unsigned int)kbdev->nr_hw_address_spaces; i++) { err = kbase_mmu_as_init(kbdev, i); if (err) break; @@ -101,12 +102,39 @@ static int kbase_device_all_as_init(struct kbase_device *kbdev) static void kbase_device_all_as_term(struct kbase_device *kbdev) { - int i; + unsigned int i; - for (i = 0; i < kbdev->nr_hw_address_spaces; i++) + for (i = 0; i < (unsigned int)kbdev->nr_hw_address_spaces; i++) kbase_mmu_as_term(kbdev, i); } +static int pcm_prioritized_process_cb(struct notifier_block *nb, unsigned long action, void *data) +{ +#if MALI_USE_CSF + + struct kbase_device *const kbdev = + container_of(nb, struct kbase_device, pcm_prioritized_process_nb); + struct pcm_prioritized_process_notifier_data *const notifier_data = data; + int ret = -EINVAL; + + switch (action) { + case ADD_PRIORITIZED_PROCESS: + if (kbasep_adjust_prioritized_process(kbdev, true, notifier_data->pid)) + ret = 0; + break; + case REMOVE_PRIORITIZED_PROCESS: + if (kbasep_adjust_prioritized_process(kbdev, false, notifier_data->pid)) + ret = 0; + break; + } + + return ret; + +#endif /* MALI_USE_CSF */ + + return 0; +} + int kbase_device_pcm_dev_init(struct kbase_device *const kbdev) { int err = 0; @@ -140,6 +168,18 @@ int kbase_device_pcm_dev_init(struct kbase_device *const kbdev) dev_info(kbdev->dev, "Priority control manager successfully loaded"); kbdev->pcm_dev = pcm_dev; + + kbdev->pcm_prioritized_process_nb = (struct notifier_block){ + .notifier_call = &pcm_prioritized_process_cb, + }; + if (pcm_dev->ops.pcm_prioritized_process_notifier_register != + NULL) { + if (pcm_dev->ops.pcm_prioritized_process_notifier_register( + pcm_dev, &kbdev->pcm_prioritized_process_nb)) + dev_warn( + kbdev->dev, + "Failed to register for changes in prioritized processes"); + } } } of_node_put(prio_ctrl_node); @@ -151,8 +191,14 @@ int kbase_device_pcm_dev_init(struct kbase_device *const kbdev) void kbase_device_pcm_dev_term(struct kbase_device *const kbdev) { - if (kbdev->pcm_dev) - module_put(kbdev->pcm_dev->owner); + struct priority_control_manager_device *const pcm_dev = kbdev->pcm_dev; + + if (pcm_dev) { + if (pcm_dev->ops.pcm_prioritized_process_notifier_unregister != NULL) + pcm_dev->ops.pcm_prioritized_process_notifier_unregister( + pcm_dev, &kbdev->pcm_prioritized_process_nb); + module_put(pcm_dev->owner); + } } #define KBASE_PAGES_TO_KIB(pages) (((unsigned int)pages) << (PAGE_SHIFT - 10)) @@ -268,7 +314,7 @@ int kbase_device_misc_init(struct kbase_device *const kbdev) if (err) goto dma_set_mask_failed; - kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces; + kbdev->nr_hw_address_spaces = (s8)kbdev->gpu_props.num_address_spaces; err = kbase_device_all_as_init(kbdev); if (err) @@ -291,8 +337,6 @@ int kbase_device_misc_init(struct kbase_device *const kbdev) #endif /* !MALI_USE_CSF */ kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); - kbdev->mmu_or_gpu_cache_op_wait_time_ms = - kbase_get_timeout_ms(kbdev, MMU_AS_INACTIVE_WAIT_TIMEOUT); mutex_init(&kbdev->kctx_list_lock); INIT_LIST_HEAD(&kbdev->kctx_list); diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c b/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c index 3b507c4fec63..da597af9c46e 100644 --- a/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device_hw.c @@ -48,7 +48,7 @@ bool kbase_is_gpu_removed(struct kbase_device *kbdev) static int busy_wait_cache_operation(struct kbase_device *kbdev, u32 irq_bit) { const ktime_t wait_loop_start = ktime_get_raw(); - const u32 wait_time_ms = kbdev->mmu_or_gpu_cache_op_wait_time_ms; + const u32 wait_time_ms = kbase_get_timeout_ms(kbdev, MMU_AS_INACTIVE_WAIT_TIMEOUT); bool completed = false; s64 diff; u32 irq_bits_to_check = irq_bit; @@ -169,7 +169,7 @@ int kbase_gpu_cache_flush_and_busy_wait(struct kbase_device *kbdev, u32 flush_op irq_mask & ~CLEAN_CACHES_COMPLETED); /* busy wait irq status to be enabled */ - ret = busy_wait_cache_operation(kbdev, (u32)CLEAN_CACHES_COMPLETED); + ret = busy_wait_cache_operation(kbdev, CLEAN_CACHES_COMPLETED); if (ret) return ret; @@ -188,7 +188,7 @@ int kbase_gpu_cache_flush_and_busy_wait(struct kbase_device *kbdev, u32 flush_op kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_COMMAND), flush_op); /* 3. Busy-wait irq status to be enabled. */ - ret = busy_wait_cache_operation(kbdev, (u32)CLEAN_CACHES_COMPLETED); + ret = busy_wait_cache_operation(kbdev, CLEAN_CACHES_COMPLETED); if (ret) return ret; @@ -301,7 +301,7 @@ void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev) int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev, unsigned int wait_timeout_ms) { - long remaining = msecs_to_jiffies(wait_timeout_ms); + long remaining = (long)msecs_to_jiffies(wait_timeout_ms); int result = 0; while (remaining && get_cache_clean_flag(kbdev)) { diff --git a/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c index 60ba9beab91c..0ba6b2d0e3d2 100644 --- a/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c +++ b/drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_csf.c @@ -146,28 +146,28 @@ const char *kbase_gpu_exception_name(u32 const exception_code) e = "ADDRESS_SIZE_FAULT_IN"; break; case CS_FAULT_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT_0: - e = "ADDRESS_SIZE_FAULT_OUT_0 at level 0"; + e = "ADDRESS_SIZE_FAULT_OUT at level 0"; break; case CS_FAULT_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT_1: - e = "ADDRESS_SIZE_FAULT_OUT_1 at level 1"; + e = "ADDRESS_SIZE_FAULT_OUT at level 1"; break; case CS_FAULT_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT_2: - e = "ADDRESS_SIZE_FAULT_OUT_2 at level 2"; + e = "ADDRESS_SIZE_FAULT_OUT at level 2"; break; case CS_FAULT_EXCEPTION_TYPE_ADDRESS_SIZE_FAULT_OUT_3: - e = "ADDRESS_SIZE_FAULT_OUT_3 at level 3"; + e = "ADDRESS_SIZE_FAULT_OUT at level 3"; break; case CS_FAULT_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_0: - e = "MEMORY_ATTRIBUTE_FAULT_0 at level 0"; + e = "MEMORY_ATTRIBUTE_FAULT at level 0"; break; case CS_FAULT_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_1: - e = "MEMORY_ATTRIBUTE_FAULT_1 at level 1"; + e = "MEMORY_ATTRIBUTE_FAULT at level 1"; break; case CS_FAULT_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_2: - e = "MEMORY_ATTRIBUTE_FAULT_2 at level 2"; + e = "MEMORY_ATTRIBUTE_FAULT at level 2"; break; case CS_FAULT_EXCEPTION_TYPE_MEMORY_ATTRIBUTE_FAULT_3: - e = "MEMORY_ATTRIBUTE_FAULT_3 at level 3"; + e = "MEMORY_ATTRIBUTE_FAULT at level 3"; break; /* Any other exception code is unknown */ default: diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.h b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.h index c56c0b67a17f..40356596163d 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.h +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.h @@ -22,6 +22,8 @@ #ifndef _MALI_KBASE_HW_ACCESS_H_ #define _MALI_KBASE_HW_ACCESS_H_ +#include + #define KBASE_REGMAP_PERM_READ (1U << 0) #define KBASE_REGMAP_PERM_WRITE (1U << 1) #define KBASE_REGMAP_WIDTH_32_BIT (1U << 2) @@ -186,4 +188,37 @@ u32 kbase_regmap_backend_init(struct kbase_device *kbdev); */ void kbase_regmap_term(struct kbase_device *kbdev); +/** + * kbase_reg_poll32_timeout - Poll a 32 bit register with timeout + * @kbdev: Kbase device pointer + * @reg_enum: Register enum + * @val: Variable for result of read + * @cond: Condition to be met + * @delay_us: Delay between each poll (in uS) + * @timeout_us: Timeout (in uS) + * @delay_before_read: If true delay for @delay_us before read + * + * Return: 0 if condition is met, -ETIMEDOUT if timed out. + */ +#define kbase_reg_poll32_timeout(kbdev, reg_enum, val, cond, delay_us, timeout_us, \ + delay_before_read) \ + read_poll_timeout_atomic(kbase_reg_read32, val, cond, delay_us, timeout_us, \ + delay_before_read, kbdev, reg_enum) + +/** + * kbase_reg_poll64_timeout - Poll a 64 bit register with timeout + * @kbdev: Kbase device pointer + * @reg_enum: Register enum + * @val: Variable for result of read + * @cond: Condition to be met + * @delay_us: Delay between each poll (in uS) + * @timeout_us: Timeout (in uS) + * @delay_before_read: If true delay for @delay_us before read + * + * Return: 0 if condition is met, -ETIMEDOUT if timed out. + */ +#define kbase_reg_poll64_timeout(kbdev, reg_enum, val, cond, delay_us, timeout_us, \ + delay_before_read) \ + read_poll_timeout_atomic(kbase_reg_read64, val, cond, delay_us, timeout_us, \ + delay_before_read, kbdev, reg_enum) #endif /* _MALI_KBASE_HW_ACCESS_H_ */ diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h index ead7ac8dfb30..97adb1322a35 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h @@ -290,10 +290,10 @@ /* THREAD_* registers */ /* THREAD_FEATURES IMPLEMENTATION_TECHNOLOGY values */ -#define THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_NOT_SPECIFIED 0 -#define THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_SILICON 1 -#define THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_FPGA 2 -#define THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_SOFTWARE 3 +#define THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_NOT_SPECIFIED 0U +#define THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_SILICON 1U +#define THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_FPGA 2U +#define THREAD_FEATURES_IMPLEMENTATION_TECHNOLOGY_SOFTWARE 3U /* End THREAD_* registers */ @@ -430,7 +430,7 @@ /* SYSC_ALLOC register */ #define SYSC_ALLOC_R_SYSC_ALLOC0_SHIFT (0) -#define SYSC_ALLOC_R_SYSC_ALLOC0_MASK ((0xF) << SYSC_ALLOC_R_SYSC_ALLOC0_SHIFT) +#define SYSC_ALLOC_R_SYSC_ALLOC0_MASK ((0xFU) << SYSC_ALLOC_R_SYSC_ALLOC0_SHIFT) #define SYSC_ALLOC_R_SYSC_ALLOC0_GET(reg_val) \ (((reg_val)&SYSC_ALLOC_R_SYSC_ALLOC0_MASK) >> SYSC_ALLOC_R_SYSC_ALLOC0_SHIFT) #define SYSC_ALLOC_R_SYSC_ALLOC0_SET(reg_val, value) \ @@ -438,7 +438,7 @@ (((value) << SYSC_ALLOC_R_SYSC_ALLOC0_SHIFT) & SYSC_ALLOC_R_SYSC_ALLOC0_MASK)) /* End of SYSC_ALLOC_R_SYSC_ALLOC0 values */ #define SYSC_ALLOC_W_SYSC_ALLOC0_SHIFT (4) -#define SYSC_ALLOC_W_SYSC_ALLOC0_MASK ((0xF) << SYSC_ALLOC_W_SYSC_ALLOC0_SHIFT) +#define SYSC_ALLOC_W_SYSC_ALLOC0_MASK ((0xFU) << SYSC_ALLOC_W_SYSC_ALLOC0_SHIFT) #define SYSC_ALLOC_W_SYSC_ALLOC0_GET(reg_val) \ (((reg_val)&SYSC_ALLOC_W_SYSC_ALLOC0_MASK) >> SYSC_ALLOC_W_SYSC_ALLOC0_SHIFT) #define SYSC_ALLOC_W_SYSC_ALLOC0_SET(reg_val, value) \ @@ -446,7 +446,7 @@ (((value) << SYSC_ALLOC_W_SYSC_ALLOC0_SHIFT) & SYSC_ALLOC_W_SYSC_ALLOC0_MASK)) /* End of SYSC_ALLOC_W_SYSC_ALLOC0 values */ #define SYSC_ALLOC_R_SYSC_ALLOC1_SHIFT (8) -#define SYSC_ALLOC_R_SYSC_ALLOC1_MASK ((0xF) << SYSC_ALLOC_R_SYSC_ALLOC1_SHIFT) +#define SYSC_ALLOC_R_SYSC_ALLOC1_MASK ((0xFU) << SYSC_ALLOC_R_SYSC_ALLOC1_SHIFT) #define SYSC_ALLOC_R_SYSC_ALLOC1_GET(reg_val) \ (((reg_val)&SYSC_ALLOC_R_SYSC_ALLOC1_MASK) >> SYSC_ALLOC_R_SYSC_ALLOC1_SHIFT) #define SYSC_ALLOC_R_SYSC_ALLOC1_SET(reg_val, value) \ @@ -454,7 +454,7 @@ (((value) << SYSC_ALLOC_R_SYSC_ALLOC1_SHIFT) & SYSC_ALLOC_R_SYSC_ALLOC1_MASK)) /* End of SYSC_ALLOC_R_SYSC_ALLOC1 values */ #define SYSC_ALLOC_W_SYSC_ALLOC1_SHIFT (12) -#define SYSC_ALLOC_W_SYSC_ALLOC1_MASK ((0xF) << SYSC_ALLOC_W_SYSC_ALLOC1_SHIFT) +#define SYSC_ALLOC_W_SYSC_ALLOC1_MASK ((0xFU) << SYSC_ALLOC_W_SYSC_ALLOC1_SHIFT) #define SYSC_ALLOC_W_SYSC_ALLOC1_GET(reg_val) \ (((reg_val)&SYSC_ALLOC_W_SYSC_ALLOC1_MASK) >> SYSC_ALLOC_W_SYSC_ALLOC1_SHIFT) #define SYSC_ALLOC_W_SYSC_ALLOC1_SET(reg_val, value) \ @@ -462,7 +462,7 @@ (((value) << SYSC_ALLOC_W_SYSC_ALLOC1_SHIFT) & SYSC_ALLOC_W_SYSC_ALLOC1_MASK)) /* End of SYSC_ALLOC_W_SYSC_ALLOC1 values */ #define SYSC_ALLOC_R_SYSC_ALLOC2_SHIFT (16) -#define SYSC_ALLOC_R_SYSC_ALLOC2_MASK ((0xF) << SYSC_ALLOC_R_SYSC_ALLOC2_SHIFT) +#define SYSC_ALLOC_R_SYSC_ALLOC2_MASK ((0xFU) << SYSC_ALLOC_R_SYSC_ALLOC2_SHIFT) #define SYSC_ALLOC_R_SYSC_ALLOC2_GET(reg_val) \ (((reg_val)&SYSC_ALLOC_R_SYSC_ALLOC2_MASK) >> SYSC_ALLOC_R_SYSC_ALLOC2_SHIFT) #define SYSC_ALLOC_R_SYSC_ALLOC2_SET(reg_val, value) \ @@ -470,7 +470,7 @@ (((value) << SYSC_ALLOC_R_SYSC_ALLOC2_SHIFT) & SYSC_ALLOC_R_SYSC_ALLOC2_MASK)) /* End of SYSC_ALLOC_R_SYSC_ALLOC2 values */ #define SYSC_ALLOC_W_SYSC_ALLOC2_SHIFT (20) -#define SYSC_ALLOC_W_SYSC_ALLOC2_MASK ((0xF) << SYSC_ALLOC_W_SYSC_ALLOC2_SHIFT) +#define SYSC_ALLOC_W_SYSC_ALLOC2_MASK ((0xFU) << SYSC_ALLOC_W_SYSC_ALLOC2_SHIFT) #define SYSC_ALLOC_W_SYSC_ALLOC2_GET(reg_val) \ (((reg_val)&SYSC_ALLOC_W_SYSC_ALLOC2_MASK) >> SYSC_ALLOC_W_SYSC_ALLOC2_SHIFT) #define SYSC_ALLOC_W_SYSC_ALLOC2_SET(reg_val, value) \ @@ -478,7 +478,7 @@ (((value) << SYSC_ALLOC_W_SYSC_ALLOC2_SHIFT) & SYSC_ALLOC_W_SYSC_ALLOC2_MASK)) /* End of SYSC_ALLOC_W_SYSC_ALLOC2 values */ #define SYSC_ALLOC_R_SYSC_ALLOC3_SHIFT (24) -#define SYSC_ALLOC_R_SYSC_ALLOC3_MASK ((0xF) << SYSC_ALLOC_R_SYSC_ALLOC3_SHIFT) +#define SYSC_ALLOC_R_SYSC_ALLOC3_MASK ((0xFU) << SYSC_ALLOC_R_SYSC_ALLOC3_SHIFT) #define SYSC_ALLOC_R_SYSC_ALLOC3_GET(reg_val) \ (((reg_val)&SYSC_ALLOC_R_SYSC_ALLOC3_MASK) >> SYSC_ALLOC_R_SYSC_ALLOC3_SHIFT) #define SYSC_ALLOC_R_SYSC_ALLOC3_SET(reg_val, value) \ @@ -486,7 +486,7 @@ (((value) << SYSC_ALLOC_R_SYSC_ALLOC3_SHIFT) & SYSC_ALLOC_R_SYSC_ALLOC3_MASK)) /* End of SYSC_ALLOC_R_SYSC_ALLOC3 values */ #define SYSC_ALLOC_W_SYSC_ALLOC3_SHIFT (28) -#define SYSC_ALLOC_W_SYSC_ALLOC3_MASK ((0xF) << SYSC_ALLOC_W_SYSC_ALLOC3_SHIFT) +#define SYSC_ALLOC_W_SYSC_ALLOC3_MASK ((0xFU) << SYSC_ALLOC_W_SYSC_ALLOC3_SHIFT) #define SYSC_ALLOC_W_SYSC_ALLOC3_GET(reg_val) \ (((reg_val)&SYSC_ALLOC_W_SYSC_ALLOC3_MASK) >> SYSC_ALLOC_W_SYSC_ALLOC3_SHIFT) #define SYSC_ALLOC_W_SYSC_ALLOC3_SET(reg_val, value) \ diff --git a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h index 8f6164ee23bf..c3d12ad04c4e 100644 --- a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h +++ b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h @@ -69,6 +69,7 @@ ENUM_OFFSET(n, DOORBELL_BLOCK_ENUM(0, regname), DOORBELL_BLOCK_ENUM(1, regname)) /* register value macros */ + /* L2_CONFIG PBHA values */ #define L2_CONFIG_PBHA_HWU_SHIFT GPU_U(12) #define L2_CONFIG_PBHA_HWU_MASK (GPU_U(0xF) << L2_CONFIG_PBHA_HWU_SHIFT) @@ -79,7 +80,8 @@ (((value) << L2_CONFIG_PBHA_HWU_SHIFT) & L2_CONFIG_PBHA_HWU_MASK)) #define PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_SHIFT (0) -#define PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_MASK ((0xFF) << PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_SHIFT) +#define PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_MASK \ + ((0xFFU) << PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_SHIFT) #define PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_GET(reg_val) \ (((reg_val)&PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_MASK) >> \ PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_SHIFT) @@ -401,16 +403,17 @@ (((value) << GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT) & GPU_FAULTSTATUS_ADDRESS_VALID_MASK)) /* GPU IRQ flags */ -#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ -#define GPU_PROTECTED_FAULT (1 << 1) /* A GPU fault has occurred in protected mode */ -#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. */ -#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ -#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down. */ -#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ -#define DOORBELL_MIRROR (1 << 18) /* Mirrors the doorbell interrupt line to the CPU */ -#define MCU_STATUS_GPU_IRQ (1 << 19) /* MCU requires attention */ +#define GPU_FAULT (1U << 0) /* A GPU Fault has occurred */ +#define GPU_PROTECTED_FAULT (1U << 1) /* A GPU fault has occurred in protected mode */ +#define RESET_COMPLETED (1U << 8) /* Set when a reset has completed. */ +#define POWER_CHANGED_SINGLE \ + (1U << 9) /* Set when a single core has finished powering up or down. */ +#define POWER_CHANGED_ALL (1U << 10) /* Set when all cores have finished powering up or down. */ +#define CLEAN_CACHES_COMPLETED (1U << 17) /* Set when a cache clean operation has completed. */ +#define DOORBELL_MIRROR (1U << 18) /* Mirrors the doorbell interrupt line to the CPU */ +#define MCU_STATUS_GPU_IRQ (1U << 19) /* MCU requires attention */ #define FLUSH_PA_RANGE_COMPLETED \ - (1 << 20) /* Set when a physical range cache clean operation has completed. */ + (1U << 20) /* Set when a physical range cache clean operation has completed. */ /* GPU_FEATURES register */ diff --git a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_macros.h b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_macros.h index 1cdd215735eb..c3bc0f3e9924 100644 --- a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_macros.h +++ b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_jm_macros.h @@ -269,13 +269,14 @@ #define GPU_COMMAND_FLUSH_CACHE_MERGE(cmd1, cmd2) ((cmd1) > (cmd2) ? (cmd1) : (cmd2)) /* IRQ flags */ -#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ -#define MULTIPLE_GPU_FAULTS (1 << 7) /* More than one GPU Fault occurred. */ -#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. */ -#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ -#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down. */ -#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ -#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ +#define GPU_FAULT (1U << 0) /* A GPU Fault has occurred */ +#define MULTIPLE_GPU_FAULTS (1U << 7) /* More than one GPU Fault occurred. */ +#define RESET_COMPLETED (1U << 8) /* Set when a reset has completed. */ +#define POWER_CHANGED_SINGLE \ + (1U << 9) /* Set when a single core has finished powering up or down. */ +#define POWER_CHANGED_ALL (1U << 10) /* Set when all cores have finished powering up or down. */ +#define PRFCNT_SAMPLE_COMPLETED (1U << 16) /* Set when a performance count sample has completed. */ +#define CLEAN_CACHES_COMPLETED (1U << 17) /* Set when a cache clean operation has completed. */ #define FLUSH_PA_RANGE_COMPLETED \ (1 << 20) /* Set when a physical range cache clean operation has completed. */ diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c index f23a5aacdbfd..d605253752ca 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c @@ -44,6 +44,9 @@ #define HWCNT_BACKEND_WATCHDOG_TIMER_INTERVAL_MS ((u32)1000) #endif /* IS_FPGA && !NO_MALI */ +/* Used to check for a sample in which all counters in the block are disabled */ +#define HWCNT_BLOCK_EMPTY_SAMPLE (2) + /** * enum kbase_hwcnt_backend_csf_dump_state - HWC CSF backend dumping states. * @@ -339,8 +342,7 @@ kbasep_hwcnt_backend_csf_cc_initial_sample(struct kbase_hwcnt_backend_csf *backe backend_csf->info->csf_if->get_gpu_cycle_count(backend_csf->info->csf_if->ctx, cycle_counts, clk_enable_map); - kbase_hwcnt_metadata_for_each_clock(enable_map->metadata, clk) - { + kbase_hwcnt_metadata_for_each_clock(enable_map->metadata, clk) { if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, clk)) backend_csf->prev_cycle_count[clk] = cycle_counts[clk]; } @@ -361,8 +363,7 @@ static void kbasep_hwcnt_backend_csf_cc_update(struct kbase_hwcnt_backend_csf *b backend_csf->info->csf_if->get_gpu_cycle_count(backend_csf->info->csf_if->ctx, cycle_counts, backend_csf->clk_enable_map); - kbase_hwcnt_metadata_for_each_clock(backend_csf->info->metadata, clk) - { + kbase_hwcnt_metadata_for_each_clock(backend_csf->info->metadata, clk) { if (kbase_hwcnt_clk_enable_map_enabled(backend_csf->clk_enable_map, clk)) { backend_csf->cycle_count_elapsed[clk] = cycle_counts[clk] - backend_csf->prev_cycle_count[clk]; @@ -384,34 +385,28 @@ static u64 kbasep_hwcnt_backend_csf_timestamp_ns(struct kbase_hwcnt_backend *bac /** kbasep_hwcnt_backend_csf_process_enable_map() - Process the enable_map to * guarantee headers are - * enabled if any counter is - * required. + * enabled. *@phys_enable_map: HWC physical enable map to be processed. */ -static void -kbasep_hwcnt_backend_csf_process_enable_map(struct kbase_hwcnt_physical_enable_map *phys_enable_map) +void kbasep_hwcnt_backend_csf_process_enable_map( + struct kbase_hwcnt_physical_enable_map *phys_enable_map) { WARN_ON(!phys_enable_map); - /* Enable header if any counter is required from user, the header is - * controlled by bit 0 of the enable mask. + /* Unconditionally enable each block header and first counter, + * the header is controlled by bit 0 of the enable mask. */ - phys_enable_map->fe_bm |= 1; + phys_enable_map->fe_bm |= 3; - if (phys_enable_map->tiler_bm) - phys_enable_map->tiler_bm |= 1; + phys_enable_map->tiler_bm |= 3; - if (phys_enable_map->mmu_l2_bm) - phys_enable_map->mmu_l2_bm |= 1; + phys_enable_map->mmu_l2_bm |= 3; - if (phys_enable_map->shader_bm) - phys_enable_map->shader_bm |= 1; + phys_enable_map->shader_bm |= 3; - if (phys_enable_map->fw_bm) - phys_enable_map->fw_bm |= 1; + phys_enable_map->fw_bm |= 3; - if (phys_enable_map->csg_bm) - phys_enable_map->csg_bm |= 1; + phys_enable_map->csg_bm |= 3; } @@ -429,7 +424,7 @@ static void kbasep_hwcnt_backend_csf_init_layout( WARN_ON(!prfcnt_info); WARN_ON(!phys_layout); - shader_core_cnt = fls64(prfcnt_info->core_mask); + shader_core_cnt = (size_t)fls64(prfcnt_info->core_mask); values_per_block = prfcnt_info->prfcnt_block_size / KBASE_HWCNT_VALUE_HW_BYTES; fw_block_cnt = div_u64(prfcnt_info->prfcnt_fw_size, prfcnt_info->prfcnt_block_size); hw_block_cnt = div_u64(prfcnt_info->prfcnt_hw_size, prfcnt_info->prfcnt_block_size); @@ -542,7 +537,8 @@ static void kbasep_hwcnt_backend_csf_update_block_state( size_t block_idx, blk_stt_t *const block_state, bool fw_in_protected_mode) { /* Offset of shader core blocks from the start of the HW blocks in the sample */ - size_t shader_core_block_offset = phys_layout->hw_block_cnt - phys_layout->shader_cnt; + size_t shader_core_block_offset = + (size_t)(phys_layout->hw_block_cnt - phys_layout->shader_cnt); bool is_shader_core_block; is_shader_core_block = block_idx >= shader_core_block_offset; @@ -608,17 +604,17 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample( const u32 *old_block = old_sample_buf; const u32 *new_block = new_sample_buf; u64 *acc_block = accum_buf; - /* Flag to indicate whether current sample is when exiting protected mode. */ + /* Flag to indicate whether current sample is exiting protected mode. */ bool exiting_protm = false; const size_t values_per_block = phys_layout->values_per_block; /* The block pointers now point to the first HW block, which is always a CSHW/front-end * block. The counter enable mask for this block can be checked to determine whether this * sample is taken after leaving protected mode - this is the only scenario where the CSHW - * block counter enable mask is all-zero. In this case, the values in this sample would not - * be meaningful, so they don't need to be accumulated. + * block counter enable mask has only the first bit set, and no others. In this case, + * the values in this sample would not be meaningful, so they don't need to be accumulated. */ - exiting_protm = !new_block[phys_layout->enable_mask_offset]; + exiting_protm = (new_block[phys_layout->enable_mask_offset] == 1); for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { const u32 old_enable_mask = old_block[phys_layout->enable_mask_offset]; @@ -629,7 +625,7 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample( &block_states[block_idx], fw_in_protected_mode); - if (new_enable_mask == 0) { + if (!(new_enable_mask & HWCNT_BLOCK_EMPTY_SAMPLE)) { /* Hardware block was unavailable or we didn't turn on * any counters. Do nothing. */ @@ -662,7 +658,7 @@ static void kbasep_hwcnt_backend_csf_accumulate_sample( * saturating at their maximum value. */ if (!clearing_samples) { - if (old_enable_mask == 0) { + if (!(old_enable_mask & HWCNT_BLOCK_EMPTY_SAMPLE)) { /* Block was previously * unavailable. Accumulate the new * counters only, as we know previous @@ -1444,8 +1440,7 @@ static int kbasep_hwcnt_backend_csf_dump_get(struct kbase_hwcnt_backend *backend return -EINVAL; /* Extract elapsed cycle count for each clock domain if enabled. */ - kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) - { + kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) { if (!kbase_hwcnt_clk_enable_map_enabled(dst_enable_map->clk_enable_map, clk)) continue; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h index da78c1f76aae..2487db272a35 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.h @@ -31,6 +31,8 @@ #include "hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h" #include "hwcnt/mali_kbase_hwcnt_watchdog_if.h" +struct kbase_hwcnt_physical_enable_map; + /** * kbase_hwcnt_backend_csf_create() - Create a CSF hardware counter backend * interface. @@ -127,6 +129,15 @@ void kbase_hwcnt_backend_csf_set_hw_availability(struct kbase_hwcnt_backend_inte size_t num_l2_slices, uint64_t shader_present_bitmap); +/** kbasep_hwcnt_backend_csf_process_enable_map() - Process the enable_map to + * guarantee headers are + * enabled if any counter is + * required. + * @phys_enable_map: HWC physical enable map to be processed. + */ +void kbasep_hwcnt_backend_csf_process_enable_map( + struct kbase_hwcnt_physical_enable_map *phys_enable_map); + /** * kbase_hwcnt_backend_csf_on_prfcnt_sample() - CSF performance counter sample * complete interrupt handler. diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c index a44651949abf..d79a99e5e89f 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c @@ -536,7 +536,7 @@ kbasep_hwcnt_backend_csf_if_fw_dump_enable(struct kbase_hwcnt_backend_csf_if_ctx (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf = (struct kbase_hwcnt_backend_csf_if_fw_ring_buf *)ring_buf; - u32 max_csg_slots; + u32 csg_mask; WARN_ON(!ctx); WARN_ON(!ring_buf); @@ -545,7 +545,7 @@ kbasep_hwcnt_backend_csf_if_fw_dump_enable(struct kbase_hwcnt_backend_csf_if_ctx kbdev = fw_ctx->kbdev; global_iface = &kbdev->csf.global_iface; - max_csg_slots = kbdev->csf.global_iface.group_num; + csg_mask = (1 << kbdev->csf.global_iface.group_num) - 1; /* Configure */ prfcnt_config = GLB_PRFCNT_CONFIG_SIZE_SET(0, fw_ring_buf->buf_count); @@ -570,7 +570,7 @@ kbasep_hwcnt_backend_csf_if_fw_dump_enable(struct kbase_hwcnt_backend_csf_if_ctx kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_CSG_EN, enable->csg_bm); /* Enable all of the CSGs by default. */ - kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_CSG_SELECT, max_csg_slots); + kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_CSG_SELECT, csg_mask); /* Configure the HWC set and buffer size */ diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c index 5156706bdf2b..7fbef163976a 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c @@ -80,34 +80,40 @@ struct kbase_hwcnt_jm_physical_layout { /** * struct kbase_hwcnt_backend_jm - Instance of a JM hardware counter backend. - * @info: Info used to create the backend. - * @kctx: KBase context used for GPU memory allocation and - * counter dumping. - * @gpu_dump_va: GPU hardware counter dump buffer virtual address. - * @cpu_dump_va: CPU mapping of gpu_dump_va. - * @vmap: Dump buffer vmap. - * @to_user_buf: HWC sample buffer for client user, size - * metadata.dump_buf_bytes. - * @enabled: True if dumping has been enabled, else false. - * @accum_all_blk_stt: Block State to accumulate on next sample, for all types - * of block. + * @info: Info used to create the backend. + * @kctx: KBase context used for GPU memory allocation and + * counter dumping. + * @gpu_dump_va: GPU hardware counter dump buffer virtual address. + * @cpu_dump_va: CPU mapping of gpu_dump_va. + * @vmap: Dump buffer vmap. + * @to_user_buf: HWC sample buffer for client user, size + * metadata.dump_buf_bytes. + * @enabled: True if dumping has been enabled, else false. + * @accum_all_blk_stt: Block State to accumulate on next sample, for all types + * of block. * @sampled_all_blk_stt: Block State to accumulate into the current sample, for * all types of block. - * @pm_core_mask: PM state sync-ed shaders core mask for the enabled - * dumping. - * @curr_config: Current allocated hardware resources to correctly map the - * source raw dump buffer to the destination dump buffer. - * @clk_enable_map: The enable map specifying enabled clock domains. - * @cycle_count_elapsed: - * Cycle count elapsed for a given sample period. - * The top clock cycle, index 0, is read directly from - * hardware, but the other clock domains need to be - * calculated with software estimation. - * @prev_cycle_count: Previous cycle count to calculate the cycle count for - * sample period. - * @rate_listener: Clock rate listener callback state. - * @ccswe_shader_cores: Shader cores cycle count software estimator. - * @phys_layout: Physical memory layout information of HWC sample buffer. + * @debug_core_mask: User-set mask of shader cores that can be used. + * @pm_core_mask: PM state sync-ed shaders core mask for the enabled + * dumping. + * @curr_config: Current allocated hardware resources to correctly map the + * source raw dump buffer to the destination dump buffer. + * @max_core_mask: Core mask of all cores allocated to the GPU (non + * virtualized platforms) or resource group (virtualized + * platforms). + * @max_l2_slices: Maximum number of L2 slices allocated to the GPU (non + * virtualized platforms) or resource group (virtualized + * platforms). + * @clk_enable_map: The enable map specifying enabled clock domains. + * @cycle_count_elapsed: Cycle count elapsed for a given sample period. + * The top clock cycle, index 0, is read directly from + * hardware, but the other clock domains need to be + * calculated with software estimation. + * @prev_cycle_count: Previous cycle count to calculate the cycle count for + * sample period. + * @rate_listener: Clock rate listener callback state. + * @ccswe_shader_cores: Shader cores cycle count software estimator. + * @phys_layout: Physical memory layout information of HWC sample buffer. */ struct kbase_hwcnt_backend_jm { const struct kbase_hwcnt_backend_jm_info *info; @@ -119,8 +125,11 @@ struct kbase_hwcnt_backend_jm { bool enabled; blk_stt_t accum_all_blk_stt; blk_stt_t sampled_all_blk_stt; + u64 debug_core_mask; u64 pm_core_mask; struct kbase_hwcnt_curr_config curr_config; + u64 max_core_mask; + size_t max_l2_slices; u64 clk_enable_map; u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS]; u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS]; @@ -355,9 +364,9 @@ kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend *backend, struct kbase_hwcnt_backend_jm *backend_jm = (struct kbase_hwcnt_backend_jm *)backend; struct kbase_context *kctx; struct kbase_device *kbdev; - struct kbase_hwcnt_physical_enable_map phys_enable_map; + struct kbase_hwcnt_physical_enable_map phys_enable_map = { 0 }; enum kbase_hwcnt_physical_set phys_counter_set; - struct kbase_instr_hwcnt_enable enable; + struct kbase_instr_hwcnt_enable enable = { 0 }; u64 timestamp_ns; if (!backend_jm || !enable_map || backend_jm->enabled || @@ -373,18 +382,21 @@ kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend *backend, kbase_hwcnt_gpu_set_to_physical(&phys_counter_set, backend_jm->info->counter_set); - enable.fe_bm = phys_enable_map.fe_bm; - enable.shader_bm = phys_enable_map.shader_bm; - enable.tiler_bm = phys_enable_map.tiler_bm; - enable.mmu_l2_bm = phys_enable_map.mmu_l2_bm; - enable.counter_set = phys_counter_set; + enable = (struct kbase_instr_hwcnt_enable) + { + .fe_bm = phys_enable_map.fe_bm, + .shader_bm = phys_enable_map.shader_bm, + .tiler_bm = phys_enable_map.tiler_bm, + .mmu_l2_bm = phys_enable_map.mmu_l2_bm, + .counter_set = phys_counter_set, #if IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) - /* The dummy model needs the CPU mapping. */ - enable.dump_buffer = (uintptr_t)backend_jm->cpu_dump_va; + /* The dummy model needs the CPU mapping. */ + .dump_buffer = (uintptr_t)backend_jm->cpu_dump_va, #else - enable.dump_buffer = backend_jm->gpu_dump_va; + .dump_buffer = backend_jm->gpu_dump_va, #endif /* CONFIG_MALI_BIFROST_NO_MALI */ - enable.dump_buffer_bytes = backend_jm->info->dump_bytes; + .dump_buffer_bytes = backend_jm->info->dump_bytes, + }; timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); @@ -397,6 +409,10 @@ kbasep_hwcnt_backend_jm_dump_enable_nolock(struct kbase_hwcnt_backend *backend, if (errcode) goto error; + backend_jm->debug_core_mask = kbase_pm_ca_get_debug_core_mask(kbdev); + backend_jm->max_l2_slices = backend_jm->info->hwcnt_gpu_info.l2_count; + backend_jm->max_core_mask = backend_jm->info->hwcnt_gpu_info.core_mask; + backend_jm->pm_core_mask = kbase_pm_ca_get_instr_core_mask(kbdev); backend_jm->enabled = true; @@ -537,8 +553,7 @@ static int kbasep_hwcnt_backend_jm_dump_request(struct kbase_hwcnt_backend *back *dump_time_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); ret = kbase_instr_hwcnt_request_dump(backend_jm->kctx); - kbase_hwcnt_metadata_for_each_clock(metadata, clk) - { + kbase_hwcnt_metadata_for_each_clock(metadata, clk) { if (!kbase_hwcnt_clk_enable_map_enabled(backend_jm->clk_enable_map, clk)) continue; @@ -620,8 +635,7 @@ static int kbasep_hwcnt_backend_jm_dump_get(struct kbase_hwcnt_backend *backend, kbasep_hwcnt_backend_jm_dump_sample(backend_jm); /* Extract elapsed cycle count for each clock domain if enabled. */ - kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) - { + kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) { if (!kbase_hwcnt_clk_enable_map_enabled(dst_enable_map->clk_enable_map, clk)) continue; @@ -645,8 +659,9 @@ static int kbasep_hwcnt_backend_jm_dump_get(struct kbase_hwcnt_backend *backend, return errcode; #endif /* CONFIG_MALI_BIFROST_NO_MALI */ errcode = kbase_hwcnt_jm_dump_get(dst, backend_jm->to_user_buf, dst_enable_map, - backend_jm->pm_core_mask, &backend_jm->curr_config, - accumulate); + backend_jm->pm_core_mask, backend_jm->debug_core_mask, + backend_jm->max_core_mask, backend_jm->max_l2_slices, + &backend_jm->curr_config, accumulate); if (errcode) return errcode; diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.c b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.c index 875643654627..5da564546608 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.c +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.c @@ -169,7 +169,7 @@ static int kbasep_hwcnt_backend_gpu_metadata_create(const struct kbase_hwcnt_gpu /* Calculate number of block instances that aren't cores */ non_core_block_count = 2 + gpu_info->l2_count; /* Calculate number of block instances that are shader cores */ - sc_block_count = fls64(gpu_info->core_mask); + sc_block_count = (size_t)fls64(gpu_info->core_mask); /* Determine the total number of cores */ core_block_count = sc_block_count; @@ -274,7 +274,8 @@ static int kbasep_hwcnt_backend_gpu_metadata_create(const struct kbase_hwcnt_gpu desc.clk_cnt = gpu_info->clk_cnt; /* The JM, Tiler, and L2s are always available, and are before cores */ - kbase_hwcnt_set_avail_mask(&desc.avail_mask, (1ull << non_core_block_count) - 1, 0); + kbase_hwcnt_set_avail_mask(&desc.avail_mask, 0, 0); + kbase_hwcnt_set_avail_mask_bits(&desc.avail_mask, 0, non_core_block_count, U64_MAX); kbase_hwcnt_set_avail_mask_bits(&desc.avail_mask, non_core_block_count, sc_block_count, gpu_info->core_mask); @@ -293,7 +294,7 @@ static size_t kbasep_hwcnt_backend_jm_dump_bytes(const struct kbase_hwcnt_gpu_in { WARN_ON(!gpu_info); - return (2 + gpu_info->l2_count + fls64(gpu_info->core_mask)) * + return (2 + gpu_info->l2_count + (size_t)fls64(gpu_info->core_mask)) * gpu_info->prfcnt_values_per_block * KBASE_HWCNT_VALUE_HW_BYTES; } @@ -372,33 +373,50 @@ void kbase_hwcnt_csf_metadata_destroy(const struct kbase_hwcnt_metadata *metadat kbase_hwcnt_metadata_destroy(metadata); } -static bool is_block_type_shader(const u64 blk_type) +bool kbase_hwcnt_is_block_type_shader(const enum kbase_hwcnt_gpu_v5_block_type blk_type) { - bool is_shader = false; - if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC || blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2 || blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC3 || blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC_UNDEFINED) - is_shader = true; + return true; - return is_shader; + return false; } -static bool is_block_type_l2_cache(const u64 blk_type) +bool kbase_hwcnt_is_block_type_memsys(const enum kbase_hwcnt_gpu_v5_block_type blk_type) { - bool is_l2_cache = false; - if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS || blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2 || blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS_UNDEFINED) - is_l2_cache = true; + return true; - return is_l2_cache; + return false; +} + +bool kbase_hwcnt_is_block_type_tiler(const enum kbase_hwcnt_gpu_v5_block_type blk_type) +{ + if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER || + blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER_UNDEFINED) + return true; + + return false; +} + +bool kbase_hwcnt_is_block_type_fe(const enum kbase_hwcnt_gpu_v5_block_type blk_type) +{ + if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE || + blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE2 || + blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE3 || + blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE_UNDEFINED) + return true; + + return false; } int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, const struct kbase_hwcnt_enable_map *dst_enable_map, u64 pm_core_mask, + u64 debug_core_mask, u64 max_core_mask, size_t max_l2_slices, const struct kbase_hwcnt_curr_config *curr_config, bool accumulate) { const struct kbase_hwcnt_metadata *metadata; @@ -406,6 +424,7 @@ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, const u64 *dump_src = src; size_t src_offset = 0; u64 core_mask = pm_core_mask; + u64 shader_present = curr_config->shader_present; /* Variables to deal with the current configuration */ size_t l2_count = 0; @@ -415,13 +434,12 @@ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, metadata = dst->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { const size_t hdr_cnt = kbase_hwcnt_metadata_block_headers_count(metadata, blk); const size_t ctr_cnt = kbase_hwcnt_metadata_block_counters_count(metadata, blk); const u64 blk_type = kbase_hwcnt_metadata_block_type(metadata, blk); - const bool is_shader_core = is_block_type_shader(blk_type); - const bool is_l2_cache = is_block_type_l2_cache(blk_type); + const bool is_shader_core = kbase_hwcnt_is_block_type_shader(blk_type); + const bool is_l2_cache = kbase_hwcnt_is_block_type_memsys(blk_type); const bool is_undefined = kbase_hwcnt_is_block_type_undefined(blk_type); blk_stt_t *dst_blk_stt = kbase_hwcnt_dump_buffer_block_state_instance(dst, blk, blk_inst); @@ -458,25 +476,52 @@ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, if (blk_valid) { bool blk_powered; - blk_stt_t current_block_state; + blk_stt_t current_block_state = 0; if (!is_shader_core) { - /* Under the current PM system, counters will only be - * enabled after all non shader core blocks are powered up. + /* The L2 block must be available at this point, or handled + * differently below. + * Every partition must have a FE and a tiler, so they + * must be implicitly available as part of the current + * configuration. */ blk_powered = true; + current_block_state |= KBASE_HWCNT_STATE_AVAILABLE; } else { /* Check the PM core mask to see if the shader core is * powered up. */ blk_powered = core_mask & 1; + + /* Set availability bits based on whether the core is + * present in both the shader_present AND the core + * mask in sysFS. The core masks are shifted to the + * right at the end of the loop so always check the + * rightmost bit. + */ + if ((shader_present & debug_core_mask) & 0x1) + current_block_state |= KBASE_HWCNT_STATE_AVAILABLE; + else { + /* If this branch is taken, the shader core may + * be: + * * in the max configuration, but not enabled + * through the sysFS core mask + * * in the max configuration, but not in the + * current configuration + * * physically not present + */ + current_block_state |= + KBASE_HWCNT_STATE_UNAVAILABLE; + } } - current_block_state = (blk_powered) ? KBASE_HWCNT_STATE_ON : - KBASE_HWCNT_STATE_OFF; /* Note: KBASE_HWCNT_STATE_OFF for non-shader cores (L2, Tiler, JM) - * is handled on this backend's dump_disable function + * is handled on this backend's dump_disable function (since + * they are considered to always be powered here). */ + current_block_state |= (blk_powered) ? KBASE_HWCNT_STATE_ON : + KBASE_HWCNT_STATE_OFF; + if (accumulate) { /* Only update existing counter values if block was powered * and valid @@ -500,10 +545,22 @@ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, kbase_hwcnt_block_state_set(dst_blk_stt, current_block_state); } + } else if (is_l2_cache && !is_undefined) { + /* Defined L2 can only reach here when the partition does not + * own it. Check that the L2 count is within the resource + * group or whole GPU's max L2 count, and if so, + * mark it as unavailable. + */ + if (l2_count <= max_l2_slices) { + kbase_hwcnt_block_state_set( + dst_blk_stt, KBASE_HWCNT_STATE_OFF | + KBASE_HWCNT_STATE_UNAVAILABLE); + } + kbase_hwcnt_dump_buffer_block_zero(dst_blk, (hdr_cnt + ctr_cnt)); } else { - /* Even though the block might be undefined, the user has enabled - * counter collection for it. We should not propagate garbage data, - * or copy/accumulate the block states. + /* Even though the block is undefined, the user has + * enabled counter collection for it. We should not propagate + * garbage data, or copy/accumulate the block states. */ if (accumulate) { /* No-op to preserve existing values */ @@ -520,8 +577,13 @@ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, /* Just increase the src_offset if the HW is available */ if (hw_res_available) src_offset += (hdr_cnt + ctr_cnt); - if (is_shader_core) - core_mask = core_mask >> 1; + if (is_shader_core) { + /* Shift each core mask right by 1 */ + core_mask >>= 1; + debug_core_mask >>= 1; + max_core_mask >>= 1; + shader_present >>= 1; + } } return 0; @@ -544,8 +606,7 @@ int kbase_hwcnt_csf_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, metadata = dst->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { const size_t hdr_cnt = kbase_hwcnt_metadata_block_headers_count(metadata, blk); const size_t ctr_cnt = kbase_hwcnt_metadata_block_counters_count(metadata, blk); const uint64_t blk_type = kbase_hwcnt_metadata_block_type(metadata, blk); @@ -650,8 +711,7 @@ void kbase_hwcnt_gpu_enable_map_to_physical(struct kbase_hwcnt_physical_enable_m metadata = src->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { const u64 blk_type = kbase_hwcnt_metadata_block_type(metadata, blk); const u64 *blk_map = kbase_hwcnt_enable_map_block_instance(src, blk, blk_inst); const size_t map_stride = @@ -749,14 +809,31 @@ void kbase_hwcnt_gpu_set_to_physical(enum kbase_hwcnt_physical_set *dst, enum kb void kbase_hwcnt_gpu_enable_map_from_physical(struct kbase_hwcnt_enable_map *dst, const struct kbase_hwcnt_physical_enable_map *src) { - const struct kbase_hwcnt_metadata *metadata; + struct kbase_hwcnt_enable_cm cm = {}; - u64 fe_bm[EM_COUNT] = { 0 }; - u64 shader_bm[EM_COUNT] = { 0 }; - u64 tiler_bm[EM_COUNT] = { 0 }; - u64 mmu_l2_bm[EM_COUNT] = { 0 }; - u64 fw_bm[EM_COUNT] = { 0 }; - u64 csg_bm[EM_COUNT] = { 0 }; + if (WARN_ON(!src) || WARN_ON(!dst)) + return; + + kbasep_hwcnt_backend_gpu_block_map_from_physical(src->fe_bm, &cm.fe_bm[EM_LO], + &cm.fe_bm[EM_HI]); + kbasep_hwcnt_backend_gpu_block_map_from_physical(src->shader_bm, &cm.shader_bm[EM_LO], + &cm.shader_bm[EM_HI]); + kbasep_hwcnt_backend_gpu_block_map_from_physical(src->tiler_bm, &cm.tiler_bm[EM_LO], + &cm.tiler_bm[EM_HI]); + kbasep_hwcnt_backend_gpu_block_map_from_physical(src->mmu_l2_bm, &cm.mmu_l2_bm[EM_LO], + &cm.mmu_l2_bm[EM_HI]); + kbasep_hwcnt_backend_gpu_block_map_from_physical(src->fw_bm, &cm.fw_bm[EM_LO], + &cm.fw_bm[EM_HI]); + kbasep_hwcnt_backend_gpu_block_map_from_physical(src->csg_bm, &cm.csg_bm[EM_LO], + &cm.csg_bm[EM_HI]); + + kbase_hwcnt_gpu_enable_map_from_cm(dst, &cm); +} + +void kbase_hwcnt_gpu_enable_map_from_cm(struct kbase_hwcnt_enable_map *dst, + const struct kbase_hwcnt_enable_cm *src) +{ + const struct kbase_hwcnt_metadata *metadata; size_t blk, blk_inst; if (WARN_ON(!src) || WARN_ON(!dst)) @@ -764,19 +841,7 @@ void kbase_hwcnt_gpu_enable_map_from_physical(struct kbase_hwcnt_enable_map *dst metadata = dst->metadata; - kbasep_hwcnt_backend_gpu_block_map_from_physical(src->fe_bm, &fe_bm[EM_LO], &fe_bm[EM_HI]); - kbasep_hwcnt_backend_gpu_block_map_from_physical(src->shader_bm, &shader_bm[EM_LO], - &shader_bm[EM_HI]); - kbasep_hwcnt_backend_gpu_block_map_from_physical(src->tiler_bm, &tiler_bm[EM_LO], - &tiler_bm[EM_HI]); - kbasep_hwcnt_backend_gpu_block_map_from_physical(src->mmu_l2_bm, &mmu_l2_bm[EM_LO], - &mmu_l2_bm[EM_HI]); - kbasep_hwcnt_backend_gpu_block_map_from_physical(src->fw_bm, &fw_bm[EM_LO], &fw_bm[EM_HI]); - kbasep_hwcnt_backend_gpu_block_map_from_physical(src->csg_bm, &csg_bm[EM_LO], - &csg_bm[EM_HI]); - - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { const u64 blk_type = kbase_hwcnt_metadata_block_type(metadata, blk); u64 *blk_map = kbase_hwcnt_enable_map_block_instance(dst, blk, blk_inst); const size_t map_stride = @@ -806,36 +871,36 @@ void kbase_hwcnt_gpu_enable_map_from_physical(struct kbase_hwcnt_enable_map *dst case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE2: fallthrough; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE3: - blk_map[map_idx] = fe_bm[map_idx]; + blk_map[map_idx] = src->fe_bm[map_idx]; break; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: - blk_map[map_idx] = tiler_bm[map_idx]; + blk_map[map_idx] = src->tiler_bm[map_idx]; break; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: fallthrough; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: fallthrough; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC3: - blk_map[map_idx] = shader_bm[map_idx]; + blk_map[map_idx] = src->shader_bm[map_idx]; break; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: fallthrough; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2: - blk_map[map_idx] = mmu_l2_bm[map_idx]; + blk_map[map_idx] = src->mmu_l2_bm[map_idx]; break; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FW: fallthrough; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FW2: fallthrough; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FW3: - blk_map[map_idx] = fw_bm[map_idx]; + blk_map[map_idx] = src->fw_bm[map_idx]; break; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_CSG: fallthrough; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_CSG2: fallthrough; case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_CSG3: - blk_map[map_idx] = csg_bm[map_idx]; + blk_map[map_idx] = src->csg_bm[map_idx]; break; default: WARN(true, "Invalid block type %llu", blk_type); @@ -855,8 +920,7 @@ void kbase_hwcnt_gpu_patch_dump_headers(struct kbase_hwcnt_dump_buffer *buf, metadata = buf->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { u64 *buf_blk = kbase_hwcnt_dump_buffer_block_instance(buf, blk, blk_inst); const u64 *blk_map = kbase_hwcnt_enable_map_block_instance(enable_map, blk, blk_inst); diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h index 2f500fdb2237..4339fddd64e2 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h @@ -138,6 +138,24 @@ struct kbase_hwcnt_physical_enable_map { u32 csg_bm; }; +/** + * struct kbase_hwcnt_enable_cm - 128-bit enable counter masks. + * @fe_bm: Front end (JM/CSHW) counters selection bitmask. + * @shader_bm: Shader counters selection bitmask. + * @tiler_bm: Tiler counters selection bitmask. + * @mmu_l2_bm: MMU_L2 counters selection bitmask. + * @fw_bm: CSF firmware counters selection bitmask. + * @csg_bm: CSF CSG counters selection bitmask. + */ +struct kbase_hwcnt_enable_cm { + u64 fe_bm[2]; + u64 shader_bm[2]; + u64 tiler_bm[2]; + u64 mmu_l2_bm[2]; + u64 fw_bm[2]; + u64 csg_bm[2]; +}; + /* * Values for Hardware Counter SET_SELECT value. * Directly passed to HW. @@ -275,16 +293,23 @@ void kbase_hwcnt_csf_metadata_destroy(const struct kbase_hwcnt_metadata *metadat * kbase_hwcnt_jm_dump_get() - Copy or accumulate enabled counters from the raw * dump buffer in src into the dump buffer * abstraction in dst. - * @dst: Non-NULL pointer to destination dump buffer. - * @src: Non-NULL pointer to source raw dump buffer, of same length - * as dump_buf_bytes in the metadata of destination dump - * buffer. - * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. - * @pm_core_mask: PM state synchronized shaders core mask with the dump. - * @curr_config: Current allocated hardware resources to correctly map the - * source raw dump buffer to the destination dump buffer. - * @accumulate: True if counters in source should be accumulated into - * destination, rather than copied. + * @dst: Non-NULL pointer to destination dump buffer. + * @src: Non-NULL pointer to source raw dump buffer, of same length + * as dump_buf_bytes in the metadata of destination dump + * buffer. + * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. + * @pm_core_mask: PM state synchronized shaders core mask with the dump. + * @debug_core_mask: User-set mask of cores to be used by the GPU. + * @max_core_mask: Core mask of all cores allocated to the GPU (non + * virtualized platforms) or resource group (virtualized + * platforms). + * @max_l2_slices: Maximum number of L2 slices allocated to the GPU (non + * virtualised platforms) or resource group (virtualized + * platforms). + * @curr_config: Current allocated hardware resources to correctly map the + * source raw dump buffer to the destination dump buffer. + * @accumulate: True if counters in source should be accumulated into + * destination, rather than copied. * * The dst and dst_enable_map MUST have been created from the same metadata as * returned from the call to kbase_hwcnt_jm_metadata_create as was used to get @@ -294,8 +319,9 @@ void kbase_hwcnt_csf_metadata_destroy(const struct kbase_hwcnt_metadata *metadat */ int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, u64 *src, const struct kbase_hwcnt_enable_map *dst_enable_map, - const u64 pm_core_mask, - const struct kbase_hwcnt_curr_config *curr_config, bool accumulate); + const u64 pm_core_mask, u64 debug_core_mask, u64 max_core_mask, + size_t max_l2_slices, const struct kbase_hwcnt_curr_config *curr_config, + bool accumulate); /** * kbase_hwcnt_csf_dump_get() - Copy or accumulate enabled counters from the raw @@ -420,4 +446,23 @@ void kbase_hwcnt_gpu_enable_map_from_physical(struct kbase_hwcnt_enable_map *dst void kbase_hwcnt_gpu_patch_dump_headers(struct kbase_hwcnt_dump_buffer *buf, const struct kbase_hwcnt_enable_map *enable_map); +bool kbase_hwcnt_is_block_type_shader(const enum kbase_hwcnt_gpu_v5_block_type blk_type); + +bool kbase_hwcnt_is_block_type_memsys(const enum kbase_hwcnt_gpu_v5_block_type blk_type); + +bool kbase_hwcnt_is_block_type_tiler(const enum kbase_hwcnt_gpu_v5_block_type blk_type); + +bool kbase_hwcnt_is_block_type_fe(const enum kbase_hwcnt_gpu_v5_block_type blk_type); +/** + * kbase_hwcnt_gpu_enable_map_from_cm() - Builds enable map abstraction from + * counter selection bitmasks. + * @dst: Non-NULL pointer to destination enable map abstraction. + * @src: Non-NULL pointer to source counter selection bitmasks. + * + * The dst must have been created from a metadata returned from a call to + * kbase_hwcnt_jm_metadata_create or kbase_hwcnt_csf_metadata_create. + */ +void kbase_hwcnt_gpu_enable_map_from_cm(struct kbase_hwcnt_enable_map *dst, + const struct kbase_hwcnt_enable_cm *src); + #endif /* _KBASE_HWCNT_GPU_H_ */ diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.c b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.c index e7f6743f1fb1..3d0ad5af7263 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.c +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.c @@ -208,8 +208,7 @@ void kbase_hwcnt_dump_buffer_zero(struct kbase_hwcnt_dump_buffer *dst, metadata = dst->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { u64 *dst_blk; size_t val_cnt; @@ -248,8 +247,7 @@ void kbase_hwcnt_dump_buffer_zero_non_enabled(struct kbase_hwcnt_dump_buffer *ds metadata = dst->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { u64 *dst_blk = kbase_hwcnt_dump_buffer_block_instance(dst, blk, blk_inst); blk_stt_t *dst_blk_stt = kbase_hwcnt_dump_buffer_block_state_instance(dst, blk, blk_inst); @@ -291,8 +289,7 @@ void kbase_hwcnt_dump_buffer_copy(struct kbase_hwcnt_dump_buffer *dst, metadata = dst->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { u64 *dst_blk; const u64 *src_blk; blk_stt_t *dst_blk_stt; @@ -312,8 +309,7 @@ void kbase_hwcnt_dump_buffer_copy(struct kbase_hwcnt_dump_buffer *dst, kbase_hwcnt_block_state_copy(dst_blk_stt, src_blk_stt); } - kbase_hwcnt_metadata_for_each_clock(metadata, clk) - { + kbase_hwcnt_metadata_for_each_clock(metadata, clk) { if (kbase_hwcnt_clk_enable_map_enabled(dst_enable_map->clk_enable_map, clk)) dst->clk_cnt_buf[clk] = src->clk_cnt_buf[clk]; } @@ -334,8 +330,7 @@ void kbase_hwcnt_dump_buffer_copy_strict(struct kbase_hwcnt_dump_buffer *dst, metadata = dst->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { u64 *dst_blk = kbase_hwcnt_dump_buffer_block_instance(dst, blk, blk_inst); const u64 *src_blk = kbase_hwcnt_dump_buffer_block_instance(src, blk, blk_inst); blk_stt_t *dst_blk_stt = @@ -358,8 +353,7 @@ void kbase_hwcnt_dump_buffer_copy_strict(struct kbase_hwcnt_dump_buffer *dst, kbase_hwcnt_block_state_set(dst_blk_stt, KBASE_HWCNT_STATE_UNKNOWN); } - kbase_hwcnt_metadata_for_each_clock(metadata, clk) - { + kbase_hwcnt_metadata_for_each_clock(metadata, clk) { bool clk_enabled = kbase_hwcnt_clk_enable_map_enabled(dst_enable_map->clk_enable_map, clk); @@ -382,8 +376,7 @@ void kbase_hwcnt_dump_buffer_accumulate(struct kbase_hwcnt_dump_buffer *dst, metadata = dst->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { u64 *dst_blk; const u64 *src_blk; blk_stt_t *dst_blk_stt; @@ -405,8 +398,7 @@ void kbase_hwcnt_dump_buffer_accumulate(struct kbase_hwcnt_dump_buffer *dst, kbase_hwcnt_block_state_accumulate(dst_blk_stt, src_blk_stt); } - kbase_hwcnt_metadata_for_each_clock(metadata, clk) - { + kbase_hwcnt_metadata_for_each_clock(metadata, clk) { if (kbase_hwcnt_clk_enable_map_enabled(dst_enable_map->clk_enable_map, clk)) dst->clk_cnt_buf[clk] += src->clk_cnt_buf[clk]; } @@ -427,8 +419,7 @@ void kbase_hwcnt_dump_buffer_accumulate_strict(struct kbase_hwcnt_dump_buffer *d metadata = dst->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { u64 *dst_blk = kbase_hwcnt_dump_buffer_block_instance(dst, blk, blk_inst); const u64 *src_blk = kbase_hwcnt_dump_buffer_block_instance(src, blk, blk_inst); const u64 *blk_em = @@ -455,8 +446,7 @@ void kbase_hwcnt_dump_buffer_accumulate_strict(struct kbase_hwcnt_dump_buffer *d kbase_hwcnt_block_state_set(dst_blk_stt, KBASE_HWCNT_STATE_UNKNOWN); } - kbase_hwcnt_metadata_for_each_clock(metadata, clk) - { + kbase_hwcnt_metadata_for_each_clock(metadata, clk) { if (kbase_hwcnt_clk_enable_map_enabled(dst_enable_map->clk_enable_map, clk)) dst->clk_cnt_buf[clk] += src->clk_cnt_buf[clk]; else @@ -477,14 +467,12 @@ void kbase_hwcnt_dump_buffer_block_state_update(struct kbase_hwcnt_dump_buffer * metadata = dst->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { if (kbase_hwcnt_metadata_block_instance_avail(metadata, blk, blk_inst) && kbase_hwcnt_enable_map_block_enabled(dst_enable_map, blk, blk_inst)) { blk_stt_t *dst_blk_stt = kbase_hwcnt_dump_buffer_block_state_instance(dst, blk, blk_inst); - /* Block is available and enabled, so update the block state */ *dst_blk_stt |= blk_stt_val; } } diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h index 16f68ead170e..c7afe173d426 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h @@ -321,7 +321,7 @@ static inline void kbase_hwcnt_cp_avail_mask(struct kbase_hwcnt_avail_mask *dst_ * placed is expected to be fully contained by the array of bitmask elements. * @length_in_bits: The length of the value being placed in the bitmask. Assumed to be no more * than 64 bits in length. - * @value: Pointer to the source value to be written into the bitmask. + * @value: The source value to be written into the bitmask. */ static inline void kbase_hwcnt_set_avail_mask_bits(struct kbase_hwcnt_avail_mask *avail_mask, size_t offset_in_bits, size_t length_in_bits, @@ -929,8 +929,7 @@ kbase_hwcnt_enable_map_any_enabled(const struct kbase_hwcnt_enable_map *enable_m if (enable_map->metadata->clk_cnt > 0 && (enable_map->clk_enable_map & clk_enable_map_mask)) return true; - kbase_hwcnt_metadata_for_each_block(enable_map->metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(enable_map->metadata, blk, blk_inst) { if (kbase_hwcnt_enable_map_block_enabled(enable_map, blk, blk_inst)) return true; } diff --git a/drivers/gpu/arm/bifrost/ipa/backend/mali_kbase_ipa_counter_common_csf.c b/drivers/gpu/arm/bifrost/ipa/backend/mali_kbase_ipa_counter_common_csf.c index 22ba78dee7d4..323dd3bceaea 100644 --- a/drivers/gpu/arm/bifrost/ipa/backend/mali_kbase_ipa_counter_common_csf.c +++ b/drivers/gpu/arm/bifrost/ipa/backend/mali_kbase_ipa_counter_common_csf.c @@ -68,7 +68,7 @@ static s64 kbase_ipa_group_energy(s32 coeff, u64 counter_value) /* Range: 0 < counter_value < 2^38 */ /* Range: -2^59 < ret < 2^59 (as -2^21 < coeff < 2^21) */ - return counter_value * (s64)coeff; + return (s64)counter_value * (s64)coeff; } /** @@ -183,7 +183,7 @@ static int calculate_coeff(struct kbase_ipa_counter_model_data *model_data, /* Range: 0 <= coeff < 2^63 */ if (total_energy >= 0) - coeff = total_energy; + coeff = (u64)total_energy; else dev_dbg(model_data->kbdev->dev, "Energy value came negative as %lld", total_energy); @@ -224,7 +224,7 @@ static int calculate_coeff(struct kbase_ipa_counter_model_data *model_data, /* Scale by user-specified integer factor. * Range: 0 <= coeff_mul < 2^43 */ - coeff_mul = coeff * model_data->scaling_factor; + coeff_mul = coeff * (u64)model_data->scaling_factor; /* The power models have results with units * mW/(MHz V^2), i.e. nW/(Hz V^2). With precision of 1/1000000, this diff --git a/drivers/gpu/arm/bifrost/ipa/backend/mali_kbase_ipa_counter_common_jm.c b/drivers/gpu/arm/bifrost/ipa/backend/mali_kbase_ipa_counter_common_jm.c index 6e2976d9bbf9..a716e15b3df3 100644 --- a/drivers/gpu/arm/bifrost/ipa/backend/mali_kbase_ipa_counter_common_jm.c +++ b/drivers/gpu/arm/bifrost/ipa/backend/mali_kbase_ipa_counter_common_jm.c @@ -211,7 +211,7 @@ int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) /* Range: 0 <= coeff < 2^57 */ if (energy > 0) - coeff = energy; + coeff = (u64)energy; /* Range: 0 <= coeff < 2^57 (because active_cycles >= 1). However, this * can be constrained further: Counter values can only be increased by @@ -244,7 +244,7 @@ int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) /* Scale by user-specified integer factor. * Range: 0 <= coeff_mul < 2^57 */ - coeff_mul = coeff * model_data->scaling_factor; + coeff_mul = coeff * (u64)model_data->scaling_factor; /* The power models have results with units * mW/(MHz V^2), i.e. nW/(Hz V^2). With precision of 1/1000000, this diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c index e3b61d5afade..d432dc8fc8e2 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c @@ -174,7 +174,7 @@ int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, const char * of_node_put(model_dt_node); if (err && dt_required) { - strncpy(addr, "", size - 1); + strscpy(addr, "", size); dev_warn(model->kbdev->dev, "Error %d, no DT entry: %s.%s = \'%s\'\n", err, model->ops->name, name, addr); err = 0; @@ -182,7 +182,7 @@ int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, const char * } else if (err && !dt_required) { origin = "default"; } else /* !err */ { - strncpy(addr, string_prop_value, size - 1); + strscpy(addr, string_prop_value, size); origin = "DT"; } diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c index 97ab8b8e39c1..e4138580de20 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c @@ -96,7 +96,7 @@ static ssize_t param_string_set(struct file *file, const char __user *user_buf, struct kbase_ipa_model_param *param = file->private_data; struct kbase_ipa_model *model = param->model; char *old_str = NULL; - ssize_t ret = count; + ssize_t ret = (ssize_t)count; size_t buf_size; int err; diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c index 71dbc27fc025..d8a356c3e8c2 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c @@ -299,6 +299,9 @@ static int kbase_simple_power_model_recalculate(struct kbase_ipa_model *model) if (!strnlen(model_data->tz_name, sizeof(model_data->tz_name))) { model_data->gpu_tz = NULL; + dev_warn(model->kbdev->dev, + "No thermal zone specified, will use the default temperature value of %u", + FALLBACK_STATIC_TEMPERATURE); } else { char tz_name[THERMAL_NAME_LENGTH]; u32 string_len = strscpy(tz_name, model_data->tz_name, sizeof(tz_name)); diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h index ddd58eb64b82..373b9b1b73b3 100644 --- a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h +++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h @@ -29,6 +29,8 @@ #include "mali_kbase_js_defs.h" +#include + /* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */ #define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1 @@ -70,29 +72,29 @@ #define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0 /* Atom has been previously soft-stopped */ -#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1 << 1) +#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1U << 1) /* Atom has been previously retried to execute */ -#define KBASE_KATOM_FLAGS_RERUN (1 << 2) +#define KBASE_KATOM_FLAGS_RERUN (1U << 2) /* Atom submitted with JOB_CHAIN_FLAG bit set in JS_CONFIG_NEXT register, helps * to disambiguate short-running job chains during soft/hard stopping of jobs */ -#define KBASE_KATOM_FLAGS_JOBCHAIN (1 << 3) +#define KBASE_KATOM_FLAGS_JOBCHAIN (1U << 3) /* Atom has been previously hard-stopped. */ -#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1 << 4) +#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1U << 4) /* Atom has caused us to enter disjoint state */ -#define KBASE_KATOM_FLAG_IN_DISJOINT (1 << 5) +#define KBASE_KATOM_FLAG_IN_DISJOINT (1U << 5) /* Atom blocked on cross-slot dependency */ -#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1 << 7) +#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1U << 7) /* Atom has fail dependency on cross-slot dependency */ -#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1 << 8) +#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1U << 8) /* Atom is currently in the list of atoms blocked on cross-slot dependencies */ -#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1 << 9) +#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1U << 9) /* Atom requires GPU to be in protected mode */ -#define KBASE_KATOM_FLAG_PROTECTED (1 << 11) +#define KBASE_KATOM_FLAG_PROTECTED (1U << 11) /* Atom has been stored in runnable_tree */ -#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1 << 12) +#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1U << 12) /* Atom is waiting for L2 caches to power up in order to enter protected mode */ -#define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1 << 13) +#define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1U << 13) /* SW related flags about types of JS_COMMAND action * NOTE: These must be masked off by JS_COMMAND_MASK @@ -131,6 +133,9 @@ * @JM_DEFAULT_JS_FREE_TIMEOUT: Maximum timeout to wait for JS_COMMAND_NEXT * to be updated on HW side so a Job Slot is * considered free. + * @KBASE_PRFCNT_ACTIVE_TIMEOUT: Waiting time for prfcnt to be ready. + * @KBASE_CLEAN_CACHE_TIMEOUT: Waiting time for cache flush to complete. + * @KBASE_AS_INACTIVE_TIMEOUT: Waiting time for MCU address space to become inactive. * @KBASE_TIMEOUT_SELECTOR_COUNT: Number of timeout selectors. * @KBASE_DEFAULT_TIMEOUT: Fallthrough in case an invalid timeout is * passed. @@ -138,7 +143,9 @@ enum kbase_timeout_selector { MMU_AS_INACTIVE_WAIT_TIMEOUT, JM_DEFAULT_JS_FREE_TIMEOUT, - + KBASE_PRFCNT_ACTIVE_TIMEOUT, + KBASE_CLEAN_CACHE_TIMEOUT, + KBASE_AS_INACTIVE_TIMEOUT, /* Must be the last in the enum */ KBASE_TIMEOUT_SELECTOR_COUNT, KBASE_DEFAULT_TIMEOUT = JM_DEFAULT_JS_FREE_TIMEOUT @@ -526,13 +533,7 @@ struct kbase_jd_atom { /* Use the functions/API defined in mali_kbase_fence.h to * when working with this sub struct */ -#if IS_ENABLED(CONFIG_SYNC_FILE) -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence_in; -#else struct dma_fence *fence_in; -#endif -#endif /* This points to the dma-buf output fence for this atom. If * this is NULL then there is no fence for this atom and the * following fields related to dma_fence may have invalid data. @@ -544,20 +545,12 @@ struct kbase_jd_atom { * regardless of the event_code of the katom (signal also on * failure). */ -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; -#else struct dma_fence *fence; -#endif /* This is the callback object that is registered for the fence_in. * The callback is invoked when the fence_in is signaled. */ -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence_cb fence_cb; -#else struct dma_fence_cb fence_cb; -#endif bool fence_cb_added; unsigned int context; @@ -857,7 +850,7 @@ struct jsctx_queue { * @current_setup: Stores the MMU configuration for this address space. */ struct kbase_as { - int number; + unsigned int number; struct workqueue_struct *pf_wq; struct work_struct work_pagefault; struct work_struct work_busfault; diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h index 8955b0473155..333ad2d2b150 100644 --- a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h +++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h @@ -906,7 +906,7 @@ static inline void kbase_js_runpool_dec_context_count(struct kbase_device *kbdev */ static inline void kbase_js_sched_all(struct kbase_device *kbdev) { - kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); + kbase_js_sched(kbdev, (1U << kbdev->gpu_props.num_job_slots) - 1U); } extern const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS]; diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h index a61861fcb677..4426bd743b4e 100644 --- a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h +++ b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h @@ -66,9 +66,12 @@ enum base_hw_issue { BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2952, + BASE_HW_ISSUE_KRAKEHW_2151, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_END }; @@ -82,7 +85,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p0 BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { @@ -92,7 +95,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p0 BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { @@ -102,129 +105,139 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tMIx_r0p1 BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tMIx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_7891, - BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, BASE_HW_ISSUE_TMIX_8343, - BASE_HW_ISSUE_TMIX_8456, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_7891, + BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, BASE_HW_ISSUE_TMIX_8343, + BASE_HW_ISSUE_TMIX_8456, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tHEx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TSIX_1792, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TSIX_1792, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, + BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tSIx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tDVx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tNOx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tGOx_r1p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TGOX_R1_1234, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tGOx[] = { BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { @@ -235,7 +248,7 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p0 BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = { @@ -246,24 +259,32 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p1 BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTRx_r0p2[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_3076, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tTRx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = { @@ -274,24 +295,32 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tNAx_r0p0 BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tNAx_r0p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_3076, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_GPU2017_1336, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tNAx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = { @@ -301,41 +330,62 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r0p0 BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r0p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r1p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBEx_r1p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tBEx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = { @@ -345,152 +395,223 @@ __attribute__((unused)) static const enum base_hw_issue base_hw_issues_lBEx_r1p0 BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END +}; + +__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r0p1[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END +}; + +__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r0p2[] = { + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tBAx_r1p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_9435, + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TTRX_2968_TTRX_3162, + BASE_HW_ISSUE_TTRX_921, + BASE_HW_ISSUE_TTRX_3414, + BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, + BASE_HW_ISSUE_TTRX_3464, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tBAx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, + BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3212, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tODx[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3212, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tGRx[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tVAx[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_1997, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_END + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r0p1[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_1997, - BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, - BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TSIX_2033, + BASE_HW_ISSUE_TTRX_1337, + BASE_HW_ISSUE_TURSEHW_1997, + BASE_HW_ISSUE_GPU2019_3878, + BASE_HW_ISSUE_TURSEHW_2716, + BASE_HW_ISSUE_GPU2019_3901, + BASE_HW_ISSUE_GPU2021PRO_290, + BASE_HW_ISSUE_TITANHW_2710, + BASE_HW_ISSUE_TITANHW_2679, + BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_TITANHW_2922, + BASE_HW_ISSUE_TITANHW_2938, + BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tTUx[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r1p0[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r1p1[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r1p2[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTUx_r1p3[] = { BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tTIx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, - BASE_HW_ISSUE_TITANHW_2952, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, + BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2952, + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTIx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, - BASE_HW_ISSUE_TITANHW_2952, BASE_HW_ISSUE_END + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, + BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2952, + BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END +}; + +__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tTIx_r0p1[] = { + BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, + BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, + BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_tKRx_r0p0[] = { + BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_END +}; + +__attribute__((unused)) static const enum base_hw_issue base_hw_issues_tKRx_r0p1[] = { BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_END }; __attribute__((unused)) static const enum base_hw_issue base_hw_issues_model_tKRx[] = { - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_END + BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, + BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_END }; diff --git a/drivers/gpu/arm/bifrost/mali_kbase.h b/drivers/gpu/arm/bifrost/mali_kbase.h index 29c032adf15f..498d53f15f9e 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase.h +++ b/drivers/gpu/arm/bifrost/mali_kbase.h @@ -26,25 +26,6 @@ #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) -#include -#endif -#include -#include -#include -#include -#include -#include - #include #include #include @@ -75,18 +56,39 @@ #include "ipa/mali_kbase_ipa.h" +#if MALI_USE_CSF +#include "csf/mali_kbase_csf.h" +#endif + #if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) #include #endif #include "mali_linux_trace.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) +#include +#endif +#include +#include +#include +#include +#include +#include + #define KBASE_DRV_NAME "mali" #define KBASE_TIMELINE_NAME KBASE_DRV_NAME ".timeline" #if MALI_USE_CSF -#include "csf/mali_kbase_csf.h" - /* Physical memory group ID for CSF user I/O. */ #define KBASE_MEM_GROUP_CSF_IO BASE_MEM_GROUP_DEFAULT @@ -176,7 +178,16 @@ unsigned long kbase_context_get_unmapped_area(struct kbase_context *kctx, const const unsigned long len, const unsigned long pgoff, const unsigned long flags); -int assign_irqs(struct kbase_device *kbdev); +/** + * kbase_get_irqs() - Get GPU interrupts from the device tree. + * + * @kbdev: The kbase device structure of the device + * + * This function must be called once only when a kbase device is initialized. + * + * Return: 0 on success. Error code (negative) on failure. + */ +int kbase_get_irqs(struct kbase_device *kbdev); int kbase_sysfs_init(struct kbase_device *kbdev); void kbase_sysfs_term(struct kbase_device *kbdev); @@ -278,7 +289,7 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a */ void kbase_jd_done_worker(struct work_struct *data); -void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp, +void kbase_jd_done(struct kbase_jd_atom *katom, unsigned int slot_nr, ktime_t *end_timestamp, kbasep_js_atom_done_code done_code); void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom); void kbase_jd_zap_context(struct kbase_context *kctx); @@ -359,7 +370,7 @@ int kbase_job_slot_softstop_start_rp(struct kbase_context *kctx, struct kbase_va * * Where possible any job in the next register is evicted before the soft-stop. */ -void kbase_job_slot_softstop(struct kbase_device *kbdev, int js, +void kbase_job_slot_softstop(struct kbase_device *kbdev, unsigned int js, struct kbase_jd_atom *target_katom); void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, unsigned int js, @@ -541,7 +552,7 @@ static inline void kbase_pm_set_gpu_lost(struct kbase_device *kbdev, bool gpu_lo const int cur_val = atomic_xchg(&kbdev->pm.gpu_lost, new_val); if (new_val != cur_val) - KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, new_val); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, (u64)new_val); } #endif @@ -643,16 +654,17 @@ int kbase_pm_force_mcu_wakeup_after_sleep(struct kbase_device *kbdev); * * Return: the atom's ID. */ -static inline int kbase_jd_atom_id(struct kbase_context *kctx, const struct kbase_jd_atom *katom) +static inline unsigned int kbase_jd_atom_id(struct kbase_context *kctx, + const struct kbase_jd_atom *katom) { - int result; + unsigned int result; KBASE_DEBUG_ASSERT(kctx); KBASE_DEBUG_ASSERT(katom); KBASE_DEBUG_ASSERT(katom->kctx == kctx); result = katom - &kctx->jctx.atoms[0]; - KBASE_DEBUG_ASSERT(result >= 0 && result <= BASE_JD_ATOM_COUNT); + KBASE_DEBUG_ASSERT(result <= BASE_JD_ATOM_COUNT); return result; } @@ -774,6 +786,22 @@ int kbase_device_pcm_dev_init(struct kbase_device *const kbdev); */ void kbase_device_pcm_dev_term(struct kbase_device *const kbdev); +#if MALI_USE_CSF + +/** + * kbasep_adjust_prioritized_process() - Adds or removes the specified PID from + * the list of prioritized processes. + * + * @kbdev: Pointer to the structure for the kbase device + * @add: True if the process should be prioritized, false otherwise + * @tgid: The process/thread group ID + * + * Return: true if the operation was successful, false otherwise + */ +bool kbasep_adjust_prioritized_process(struct kbase_device *kbdev, bool add, uint32_t tgid); + +#endif /* MALI_USE_CSF */ + /** * KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD - If a job is soft stopped * and the number of contexts is >= this value it is reported as a disjoint event @@ -793,9 +821,9 @@ void kbase_device_pcm_dev_term(struct kbase_device *const kbdev); * * Return: sampled value of kfile::fops_count. */ -static inline u32 kbase_file_fops_count(struct kbase_file *kfile) +static inline int kbase_file_fops_count(struct kbase_file *kfile) { - u32 fops_count; + int fops_count; spin_lock(&kfile->lock); fops_count = kfile->fops_count; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h index b07207ec524f..54c60e48feae 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.h @@ -22,6 +22,8 @@ #ifndef _KBASE_AS_FAULT_DEBUG_FS_H #define _KBASE_AS_FAULT_DEBUG_FS_H +#include + /** * kbase_as_fault_debugfs_init() - Add debugfs files for reporting page faults * @@ -35,7 +37,7 @@ void kbase_as_fault_debugfs_init(struct kbase_device *kbdev); * @kbdev: Pointer to kbase_device * @as_no: The address space the fault occurred on */ -static inline void kbase_as_fault_debugfs_new(struct kbase_device *kbdev, int as_no) +static inline void kbase_as_fault_debugfs_new(struct kbase_device *kbdev, unsigned int as_no) { #if IS_ENABLED(CONFIG_DEBUG_FS) #ifdef CONFIG_MALI_BIFROST_DEBUG diff --git a/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c index 4675025baaf8..92fb703f7f00 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.c @@ -23,7 +23,8 @@ * Cache Policy API. */ -#include "mali_kbase_cache_policy.h" +#include +#include /* * The output flags should be a combination of the following values: diff --git a/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h index 1d9f00c560e7..d3d47d14edda 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_cache_policy.h @@ -26,8 +26,7 @@ #ifndef _KBASE_CACHE_POLICY_H_ #define _KBASE_CACHE_POLICY_H_ -#include "mali_kbase.h" -#include +#include /** * kbase_cache_enabled - Choose the cache policy for a specific region diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ccswe.c b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.c index c4acbf6881f3..a3c927d13e6f 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_ccswe.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.c @@ -32,13 +32,14 @@ static u64 kbasep_ccswe_cycle_at_no_lock(struct kbase_ccswe *self, u64 timestamp lockdep_assert_held(&self->access); - diff_ns = timestamp_ns - self->timestamp_ns; + diff_ns = (s64)(timestamp_ns - self->timestamp_ns); gpu_freq = diff_ns > 0 ? self->gpu_freq : self->prev_gpu_freq; diff_s = div_s64(diff_ns, NSEC_PER_SEC); diff_ns -= diff_s * NSEC_PER_SEC; - return self->cycles_elapsed + diff_s * gpu_freq + div_s64(diff_ns * gpu_freq, NSEC_PER_SEC); + return self->cycles_elapsed + (u64)diff_s * gpu_freq + + (u64)div_s64(diff_ns * gpu_freq, NSEC_PER_SEC); } void kbase_ccswe_init(struct kbase_ccswe *self) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ccswe.h b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.h index ce148ed537c5..2b27bafcf7be 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_ccswe.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_ccswe.h @@ -22,6 +22,7 @@ #ifndef _KBASE_CCSWE_H_ #define _KBASE_CCSWE_H_ +#include #include /** diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config.h b/drivers/gpu/arm/bifrost/mali_kbase_config.h index d5dd49055b00..7233e2dd3920 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_config.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_config.h @@ -26,9 +26,9 @@ #ifndef _KBASE_CONFIG_H_ #define _KBASE_CONFIG_H_ -#include #include -#include + +#include /* Forward declaration of struct kbase_device */ struct kbase_device; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h index 9dc134373dc3..20003c852863 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h @@ -219,6 +219,19 @@ enum { #define KCPU_FENCE_SIGNAL_TIMEOUT_CYCLES (1000000000ull) #endif +/* Timeout for polling the GPU in clock cycles. + * + * Based on 10s timeout based on original MAX_LOOPS value. + */ +#define IPA_INACTIVE_TIMEOUT_CYCLES (1000000000ull) + +/* Timeout for polling the GPU for the MCU status in clock cycles. + * + * Based on 120s timeout based on original MAX_LOOPS value. + */ +#define CSF_FIRMWARE_STOP_TIMEOUT_CYCLES (12000000000ull) + + /* Waiting timeout for task execution on an endpoint. Based on the * DEFAULT_PROGRESS_TIMEOUT. * @@ -250,6 +263,24 @@ enum { #endif /* !MALI_USE_CSF */ +/* Timeout for polling the GPU PRFCNT_ACTIVE bit in clock cycles. + * + * Based on 120s timeout at 100MHz, based on original MAX_LOOPS value. + */ +#define KBASE_PRFCNT_ACTIVE_TIMEOUT_CYCLES (12000000000ull) + +/* Timeout for polling the GPU for a cache flush in clock cycles. + * + * Based on 120ms timeout at 100MHz, based on original MAX_LOOPS value. + */ +#define KBASE_CLEAN_CACHE_TIMEOUT_CYCLES (12000000ull) + +/* Timeout for polling the GPU for an AS command to complete in clock cycles. + * + * Based on 120s timeout at 100MHz, based on original MAX_LOOPS value. + */ +#define KBASE_AS_INACTIVE_TIMEOUT_CYCLES (12000000000ull) + /* Default timeslice that a context is scheduled in for, in nanoseconds. * * When a context has used up this amount of time across its jobs, it is diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c index 6700337b447d..237a3b829be9 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c @@ -129,7 +129,7 @@ * @minor: Kernel minor version */ #define KBASE_API_VERSION(major, minor) \ - ((((major)&0xFFF) << 20) | (((minor)&0xFFF) << 8) | ((0 & 0xFF) << 0)) + ((((major)&0xFFFU) << 20U) | (((minor)&0xFFFU) << 8U) | ((0U & 0xFFU) << 0U)) /** * struct mali_kbase_capability_def - kbase capabilities table @@ -526,26 +526,28 @@ static struct kbase_device *to_kbase_device(struct device *dev) return dev_get_drvdata(dev); } -int assign_irqs(struct kbase_device *kbdev) +/** + * get_irqs - Get interrupts information from the device tree. + * + * @kbdev: Kbase device. + * @pdev: Platform device of the kbase device + * + * Read interrupt number and flag for 'JOB', 'MMU' and 'GPU' interrupts + * from the device tree and fill them into the struct of the kbase device. + * + * Return: 0 on successful reading of all the entries JOB, MMU and GPU interrupts. + * -EINVAL on failure for all other cases. + * + */ +static int get_irqs(struct kbase_device *kbdev, struct platform_device *pdev) { + int i; static const char *const irq_names_caps[] = { "JOB", "MMU", "GPU" }; -#if IS_ENABLED(CONFIG_OF) - static const char *const irq_names[] = { "job", "mmu", "gpu" }; -#endif - - struct platform_device *pdev; - size_t i; - - if (!kbdev) - return -ENODEV; - - pdev = to_platform_device(kbdev->dev); - for (i = 0; i < ARRAY_SIZE(irq_names_caps); i++) { + struct irq_data *irqdata; int irq; -#if IS_ENABLED(CONFIG_OF) /* We recommend using Upper case for the irq names in dts, but if * there are devices in the world using Lower case then we should * avoid breaking support for them. So try using names in Upper case @@ -553,24 +555,42 @@ int assign_irqs(struct kbase_device *kbdev) * we assume there is no IRQ resource specified for the GPU. */ irq = platform_get_irq_byname(pdev, irq_names_caps[i]); - if (irq < 0) - irq = platform_get_irq_byname(pdev, irq_names[i]); -#else - irq = platform_get_irq(pdev, i); -#endif /* CONFIG_OF */ - if (irq < 0) { - dev_err(kbdev->dev, "No IRQ resource '%s'\n", irq_names_caps[i]); - return irq; + static const char *const irq_names[] = { "job", "mmu", "gpu" }; + + irq = platform_get_irq_byname(pdev, irq_names[i]); } - kbdev->irqs[i].irq = irq; - kbdev->irqs[i].flags = irqd_get_trigger_type(irq_get_irq_data(irq)); + if (irq < 0) + return irq; + + kbdev->irqs[i].irq = (u32)irq; + irqdata = irq_get_irq_data((unsigned int)irq); + if (likely(irqdata)) + kbdev->irqs[i].flags = irqd_get_trigger_type(irqdata); + else + return -EINVAL; + + kbdev->nr_irqs++; } return 0; } + +int kbase_get_irqs(struct kbase_device *kbdev) +{ + int result; + struct platform_device *pdev = to_platform_device(kbdev->dev); + + kbdev->nr_irqs = 0; + result = get_irqs(kbdev, pdev); + if (result) + dev_err(kbdev->dev, "Invalid or No interrupt resources"); + + return result; +} + /* Find a particular kbase device (as specified by minor number), or find the "first" device if -1 is specified */ struct kbase_device *kbase_find_device(int minor) { @@ -619,19 +639,19 @@ static ssize_t write_ctx_infinite_cache(struct file *f, const char __user *ubuf, else kbase_ctx_flag_clear(kctx, KCTX_INFINITE_CACHE); - return size; + return (ssize_t)size; } static ssize_t read_ctx_infinite_cache(struct file *f, char __user *ubuf, size_t size, loff_t *off) { struct kbase_context *kctx = f->private_data; char buf[32]; - int count; + size_t count; bool value; value = kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE); - count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); + count = (size_t)scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); return simple_read_from_buffer(ubuf, size, off, buf, count); } @@ -670,19 +690,19 @@ static ssize_t write_ctx_force_same_va(struct file *f, const char __user *ubuf, kbase_ctx_flag_clear(kctx, KCTX_FORCE_SAME_VA); } - return size; + return (ssize_t)size; } static ssize_t read_ctx_force_same_va(struct file *f, char __user *ubuf, size_t size, loff_t *off) { struct kbase_context *kctx = f->private_data; char buf[32]; - int count; + size_t count; bool value; value = kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA); - count = scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); + count = (size_t)scnprintf(buf, sizeof(buf), "%s\n", value ? "Y" : "N"); return simple_read_from_buffer(ubuf, size, off, buf, count); } @@ -760,7 +780,7 @@ static int kbase_open(struct inode *inode, struct file *filp) struct kbase_file *kfile; int ret = 0; - kbdev = kbase_find_device(iminor(inode)); + kbdev = kbase_find_device((int)iminor(inode)); if (!kbdev) return -ENODEV; @@ -902,7 +922,7 @@ static int kbase_api_get_gpuprops(struct kbase_file *kfile, } if (get_props->size == 0) - return kprops->prop_buffer_size; + return (int)kprops->prop_buffer_size; if (get_props->size < kprops->prop_buffer_size) return -EINVAL; @@ -910,7 +930,7 @@ static int kbase_api_get_gpuprops(struct kbase_file *kfile, kprops->prop_buffer_size); if (err) return -EFAULT; - return kprops->prop_buffer_size; + return (int)kprops->prop_buffer_size; } #if !MALI_USE_CSF @@ -1112,8 +1132,8 @@ static int kbase_api_get_cpu_gpu_timeinfo(struct kbase_context *kctx, timeinfo->out.cycle_counter = cycle_cnt; if (flags & BASE_TIMEINFO_MONOTONIC_FLAG) { - timeinfo->out.sec = ts.tv_sec; - timeinfo->out.nsec = ts.tv_nsec; + timeinfo->out.sec = (u64)ts.tv_sec; + timeinfo->out.nsec = (u32)ts.tv_nsec; } kbase_pm_context_idle(kctx->kbdev); @@ -1140,14 +1160,14 @@ static int kbase_api_get_ddk_version(struct kbase_context *kctx, struct kbase_ioctl_get_ddk_version *version) { int ret; - uint len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); + int len = sizeof(KERNEL_SIDE_DDK_VERSION_STRING); CSTD_UNUSED(kctx); if (version->version_buffer == 0) return len; - if (version->size < len) + if (version->size < (u32)len) return -EOVERFLOW; ret = copy_to_user(u64_to_user_ptr(version->version_buffer), KERNEL_SIDE_DDK_VERSION_STRING, @@ -2193,7 +2213,7 @@ static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, lof struct kbase_file *const kfile = filp->private_data; struct kbase_context *kctx; struct base_jd_event_v2 uevent; - int out_count = 0; + size_t out_count = 0; ssize_t err = 0; CSTD_UNUSED(f_pos); @@ -2431,13 +2451,15 @@ static ssize_t power_policy_show(struct device *dev, struct device_attribute *at for (i = 0; i < policy_count && ret < (ssize_t)PAGE_SIZE; i++) { if (policy_list[i] == current_policy) - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "[%s] ", + policy_list[i]->name); else - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "%s ", + policy_list[i]->name); } if (ret < (ssize_t)PAGE_SIZE - 1) { - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "\n"); } else { buf[PAGE_SIZE - 2] = '\n'; buf[PAGE_SIZE - 1] = '\0'; @@ -2494,7 +2516,7 @@ static ssize_t power_policy_store(struct device *dev, struct device_attribute *a kbase_pm_set_policy(kbdev, new_policy); - return count; + return (ssize_t)count; } /* @@ -2533,22 +2555,22 @@ static ssize_t core_mask_show(struct device *dev, struct device_attribute *attr, spin_lock_irqsave(&kbdev->hwaccess_lock, flags); #if MALI_USE_CSF - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "Current debug core mask : 0x%llX\n", + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current debug core mask : 0x%llX\n", kbdev->pm.debug_core_mask); - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "Current desired core mask : 0x%llX\n", - kbase_pm_ca_get_core_mask(kbdev)); - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "Current in use core mask : 0x%llX\n", - kbdev->pm.backend.shaders_avail); + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), + "Current desired core mask : 0x%llX\n", kbase_pm_ca_get_core_mask(kbdev)); + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), + "Current in use core mask : 0x%llX\n", kbdev->pm.backend.shaders_avail); #else - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "Current core mask (JS0) : 0x%llX\n", + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current core mask (JS0) : 0x%llX\n", kbdev->pm.debug_core_mask[0]); - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "Current core mask (JS1) : 0x%llX\n", + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current core mask (JS1) : 0x%llX\n", kbdev->pm.debug_core_mask[1]); - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "Current core mask (JS2) : 0x%llX\n", + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Current core mask (JS2) : 0x%llX\n", kbdev->pm.debug_core_mask[2]); #endif /* MALI_USE_CSF */ - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "Available core mask : 0x%llX\n", + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "Available core mask : 0x%llX\n", kbdev->gpu_props.shader_present); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -2581,7 +2603,7 @@ static ssize_t core_mask_store(struct device *dev, struct device_attribute *attr #endif /* MALI_USE_CSF */ int items; - ssize_t err = count; + ssize_t err = (ssize_t)count; unsigned long flags; u64 shader_present; @@ -2734,7 +2756,7 @@ static ssize_t soft_job_timeout_store(struct device *dev, struct device_attribut atomic_set(&kbdev->js_data.soft_job_timeout_ms, soft_job_timeout_ms); - return count; + return (ssize_t)count; } /** @@ -2769,14 +2791,14 @@ static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, int u32 old_ticks) { if (timeout_ms > 0) { - u64 ticks = timeout_ms * 1000000ULL; + u64 ticks = (u64)timeout_ms * 1000000ULL; do_div(ticks, kbdev->js_data.scheduling_period_ns); if (!ticks) return 1; return ticks; } else if (timeout_ms < 0) { - return default_ticks; + return (u32)default_ticks; } else { return old_ticks; } @@ -2860,7 +2882,7 @@ static ssize_t js_timeouts_store(struct device *dev, struct device_attribute *at spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return count; + return (ssize_t)count; } dev_err(kbdev->dev, @@ -3042,7 +3064,7 @@ static ssize_t js_scheduling_period_store(struct device *dev, struct device_attr dev_dbg(kbdev->dev, "JS scheduling period: %dms\n", js_scheduling_period); - return count; + return (ssize_t)count; } /** @@ -3103,7 +3125,7 @@ static ssize_t js_softstop_always_store(struct device *dev, struct device_attrib kbdev->js_data.softstop_always = (bool)softstop_always; dev_dbg(kbdev->dev, "Support for softstop on a single context: %s\n", (kbdev->js_data.softstop_always) ? "Enabled" : "Disabled"); - return count; + return (ssize_t)count; } static ssize_t js_softstop_always_show(struct device *dev, struct device_attribute *attr, @@ -3192,7 +3214,8 @@ static ssize_t debug_command_show(struct device *dev, struct device_attribute *a return -ENODEV; for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT && ret < (ssize_t)PAGE_SIZE; i++) - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s\n", debug_commands[i].str); + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "%s\n", + debug_commands[i].str); if (ret >= (ssize_t)PAGE_SIZE) { buf[PAGE_SIZE - 2] = '\n'; @@ -3234,7 +3257,7 @@ static ssize_t debug_command_store(struct device *dev, struct device_attribute * for (i = 0; i < KBASEP_DEBUG_COMMAND_COUNT; i++) { if (sysfs_streq(debug_commands[i].str, buf)) { debug_commands[i].func(kbdev); - return count; + return (ssize_t)count; } } @@ -3288,6 +3311,7 @@ static ssize_t gpuinfo_show(struct device *dev, struct device_attribute *attr, c { .id = GPU_ID_PRODUCT_LODX, .name = "Mali-G610" }, { .id = GPU_ID_PRODUCT_TGRX, .name = "Mali-G510" }, { .id = GPU_ID_PRODUCT_TVAX, .name = "Mali-G310" }, + { .id = GPU_ID_PRODUCT_LTUX, .name = "Mali-G615" }, { .id = GPU_ID_PRODUCT_LTIX, .name = "Mali-G620" }, { .id = GPU_ID_PRODUCT_TKRX, .name = "Mali-TKRX" }, { .id = GPU_ID_PRODUCT_LKRX, .name = "Mali-LKRX" }, @@ -3353,6 +3377,7 @@ static ssize_t gpuinfo_show(struct device *dev, struct device_attribute *attr, c dev_dbg(kbdev->dev, "GPU ID_Name: %s (ID: 0x%x), nr_cores(%u)\n", product_name, product_id, nr_cores); } + #endif /* MALI_USE_CSF */ return scnprintf(buf, PAGE_SIZE, "%s %d cores r%dp%d 0x%08X\n", product_name, @@ -3393,10 +3418,10 @@ static ssize_t dvfs_period_store(struct device *dev, struct device_attribute *at return -EINVAL; } - kbdev->pm.dvfs_period = dvfs_period; + kbdev->pm.dvfs_period = (u32)dvfs_period; dev_dbg(kbdev->dev, "DVFS period: %dms\n", dvfs_period); - return count; + return (ssize_t)count; } /** @@ -3532,7 +3557,7 @@ static ssize_t pm_poweroff_store(struct device *dev, struct device_attribute *at if (poweroff_gpu_ticks != 0) dev_warn(kbdev->dev, "Separate GPU poweroff delay no longer supported.\n"); - return count; + return (ssize_t)count; } /** @@ -3618,7 +3643,7 @@ static ssize_t reset_timeout_store(struct device *dev, struct device_attribute * kbdev->reset_timeout_ms = reset_timeout; dev_dbg(kbdev->dev, "Reset timeout: %ums\n", reset_timeout); - return count; + return (ssize_t)count; } /** @@ -3835,13 +3860,13 @@ static DEVICE_ATTR_RW(lp_mem_pool_max_size); /** * show_simplified_mem_pool_max_size - Show the maximum size for the memory - * pool 0 of small (4KiB) pages. + * pool 0 of small (4KiB/16KiB/64KiB) pages. * @dev: The device this sysfs file is for. * @attr: The attributes of the sysfs file. * @buf: The output buffer to receive the max size. * * This function is called to get the maximum size for the memory pool 0 of - * small (4KiB) pages. It is assumed that the maximum size value is same for + * small pages. It is assumed that the maximum size value is same for * all the pools. * * Return: The number of bytes output to @buf. @@ -3862,14 +3887,14 @@ static ssize_t show_simplified_mem_pool_max_size(struct device *dev, struct devi /** * set_simplified_mem_pool_max_size - Set the same maximum size for all the - * memory pools of small (4KiB) pages. + * memory pools of small (4KiB/16KiB/64KiB) pages. * @dev: The device with sysfs file is for * @attr: The attributes of the sysfs file * @buf: The value written to the sysfs file * @count: The number of bytes written to the sysfs file * * This function is called to set the same maximum size for all the memory - * pools of small (4KiB) pages. + * pools of small pages. * * Return: The number of bytes output to @buf. */ @@ -3878,7 +3903,7 @@ static ssize_t set_simplified_mem_pool_max_size(struct device *dev, struct devic { struct kbase_device *const kbdev = to_kbase_device(dev); unsigned long new_size; - int gid; + size_t gid; int err; CSTD_UNUSED(attr); @@ -3893,7 +3918,7 @@ static ssize_t set_simplified_mem_pool_max_size(struct device *dev, struct devic for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) kbase_mem_pool_debugfs_set_max_size(kbdev->mem_pools.small, gid, (size_t)new_size); - return count; + return (ssize_t)count; } static DEVICE_ATTR(max_size, 0600, show_simplified_mem_pool_max_size, @@ -3945,7 +3970,7 @@ static ssize_t set_simplified_lp_mem_pool_max_size(struct device *dev, { struct kbase_device *const kbdev = to_kbase_device(dev); unsigned long new_size; - int gid; + size_t gid; int err; CSTD_UNUSED(attr); @@ -3960,7 +3985,7 @@ static ssize_t set_simplified_lp_mem_pool_max_size(struct device *dev, for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) kbase_mem_pool_debugfs_set_max_size(kbdev->mem_pools.large, gid, (size_t)new_size); - return count; + return (ssize_t)count; } static DEVICE_ATTR(lp_max_size, 0600, show_simplified_lp_mem_pool_max_size, @@ -3968,15 +3993,15 @@ static DEVICE_ATTR(lp_max_size, 0600, show_simplified_lp_mem_pool_max_size, /** * show_simplified_ctx_default_max_size - Show the default maximum size for the - * memory pool 0 of small (4KiB) pages. + * memory pool 0 of small (4KiB/16KiB/64KiB) pages. * @dev: The device this sysfs file is for. * @attr: The attributes of the sysfs file. * @buf: The output buffer to receive the pool size. * * This function is called to get the default ctx maximum size for the memory - * pool 0 of small (4KiB) pages. It is assumed that maximum size value is same + * pool 0 of small pages. It is assumed that maximum size value is same * for all the pools. The maximum size for the pool of large (2MiB) pages will - * be same as max size of the pool of small (4KiB) pages in terms of bytes. + * be same as max size of the pool of small pages in terms of bytes. * * Return: The number of bytes output to @buf. */ @@ -4033,7 +4058,7 @@ static ssize_t set_simplified_ctx_default_max_size(struct device *dev, kbase_mem_pool_group_config_set_max_size(&kbdev->mem_pool_defaults, (size_t)new_size); - return count; + return (ssize_t)count; } static DEVICE_ATTR(ctx_default_max_size, 0600, show_simplified_ctx_default_max_size, @@ -4103,7 +4128,7 @@ static ssize_t js_ctx_scheduling_mode_store(struct device *dev, struct device_at } if (new_js_ctx_scheduling_mode == kbdev->js_ctx_scheduling_mode) - return count; + return (ssize_t)count; mutex_lock(&kbdev->kctx_list_lock); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -4120,7 +4145,7 @@ static ssize_t js_ctx_scheduling_mode_store(struct device *dev, struct device_at dev_dbg(kbdev->dev, "JS ctx scheduling mode: %u\n", new_js_ctx_scheduling_mode); - return count; + return (ssize_t)count; } static DEVICE_ATTR_RW(js_ctx_scheduling_mode); @@ -4176,7 +4201,7 @@ static ssize_t update_serialize_jobs_setting(struct kbase_device *kbdev, const c return -EINVAL; } - return count; + return (ssize_t)count; } #if IS_ENABLED(CONFIG_DEBUG_FS) @@ -4291,15 +4316,15 @@ static ssize_t show_serialize_jobs_sysfs(struct device *dev, struct device_attri for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { if (kbdev->serialize_jobs == serialize_jobs_settings[i].setting) - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s]", + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "[%s]", serialize_jobs_settings[i].name); else - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "%s ", serialize_jobs_settings[i].name); } if (ret < (ssize_t)(PAGE_SIZE - 1)) { - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); + ret += scnprintf(buf + ret, (size_t)(PAGE_SIZE - ret), "\n"); } else { buf[PAGE_SIZE - 2] = '\n'; buf[PAGE_SIZE - 1] = '\0'; @@ -4691,7 +4716,7 @@ int power_control_init(struct kbase_device *kbdev) * operating with a partial initialization of clocks. */ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { - kbdev->clocks[i] = of_clk_get(kbdev->dev->of_node, i); + kbdev->clocks[i] = of_clk_get(kbdev->dev->of_node, (int)i); if (IS_ERR(kbdev->clocks[i])) { err = PTR_ERR(kbdev->clocks[i]); kbdev->clocks[i] = NULL; @@ -5225,6 +5250,93 @@ early_exit: #if MALI_USE_CSF + +bool kbasep_adjust_prioritized_process(struct kbase_device *kbdev, bool add, uint32_t tgid) +{ + struct kbase_context *kctx; + bool found_contexts = false; + + mutex_lock(&kbdev->kctx_list_lock); + list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { + if (kctx->tgid == tgid) { + if (add) + dev_dbg(kbdev->dev, + "Adding context %pK of process %u to prioritized list\n", + (void *)kctx, tgid); + else + dev_dbg(kbdev->dev, + "Removing context %pK of process %u from prioritized list\n", + (void *)kctx, tgid); + atomic_set(&kctx->prioritized, add); + found_contexts = true; + } + } + mutex_unlock(&kbdev->kctx_list_lock); + + if (found_contexts) + kbase_csf_scheduler_kick(kbdev); + + return found_contexts; +} + +static ssize_t add_prioritized_process_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + unsigned int tgid; + + CSTD_UNUSED(attr); + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtouint(buf, 0, &tgid); + if (ret || tgid == 0) { + dev_err(kbdev->dev, "Invalid PID specified\n"); + return -EINVAL; + } + + if (unlikely(!kbasep_adjust_prioritized_process(kbdev, true, tgid))) { + dev_err(kbdev->dev, "Non-existent PID specified\n"); + return -EINVAL; + } + + return count; +} + +static DEVICE_ATTR_WO(add_prioritized_process); + +static ssize_t remove_prioritized_process_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct kbase_device *kbdev; + int ret; + unsigned int tgid; + + CSTD_UNUSED(attr); + + kbdev = to_kbase_device(dev); + if (!kbdev) + return -ENODEV; + + ret = kstrtouint(buf, 0, &tgid); + if (ret || tgid == 0) { + dev_err(kbdev->dev, "Invalid PID specified\n"); + return -EINVAL; + } + + if (unlikely(!kbasep_adjust_prioritized_process(kbdev, false, tgid))) { + dev_err(kbdev->dev, "Non-existent PID specified\n"); + return -EINVAL; + } + + return count; +} + +static DEVICE_ATTR_WO(remove_prioritized_process); + /** * csg_scheduling_period_store - Store callback for the csg_scheduling_period * sysfs file. @@ -5264,7 +5376,7 @@ static ssize_t csg_scheduling_period_store(struct device *dev, struct device_att dev_dbg(kbdev->dev, "CSG scheduling period: %ums\n", csg_scheduling_period); kbase_csf_scheduler_unlock(kbdev); - return count; + return (ssize_t)count; } /** @@ -5328,16 +5440,16 @@ static ssize_t fw_timeout_store(struct device *dev, struct device_attribute *att "Couldn't process fw_timeout write operation.\n" "Use format 'fw_timeout_ms', and fw_timeout_ms > 0\n" "Default fw_timeout: %u", - kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_PING_TIMEOUT)); + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); return -EINVAL; } kbase_csf_scheduler_lock(kbdev); - kbdev->csf.fw_timeout_ms = fw_timeout; + kbase_device_set_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT, fw_timeout); kbase_csf_scheduler_unlock(kbdev); dev_dbg(kbdev->dev, "Firmware timeout: %ums\n", fw_timeout); - return count; + return (ssize_t)count; } /** @@ -5361,7 +5473,7 @@ static ssize_t fw_timeout_show(struct device *dev, struct device_attribute *attr if (!kbdev) return -ENODEV; - ret = scnprintf(buf, PAGE_SIZE, "%u\n", kbdev->csf.fw_timeout_ms); + ret = scnprintf(buf, PAGE_SIZE, "%u\n", kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); return ret; } @@ -5387,7 +5499,7 @@ static ssize_t idle_hysteresis_time_store(struct device *dev, struct device_attr const char *buf, size_t count) { struct kbase_device *kbdev; - u32 dur = 0; + u32 dur_us = 0; CSTD_UNUSED(attr); @@ -5395,7 +5507,7 @@ static ssize_t idle_hysteresis_time_store(struct device *dev, struct device_attr if (!kbdev) return -ENODEV; - if (kstrtou32(buf, 0, &dur)) { + if (kstrtou32(buf, 0, &dur_us)) { dev_err(kbdev->dev, "Couldn't process idle_hysteresis_time write operation.\n" "Use format \n"); return -EINVAL; @@ -5404,9 +5516,9 @@ static ssize_t idle_hysteresis_time_store(struct device *dev, struct device_attr /* In sysFs, The unit of the input value of idle_hysteresis_time is us. * But the unit of the input parameter of this function is ns, so multiply by 1000 */ - kbase_csf_firmware_set_gpu_idle_hysteresis_time(kbdev, dur * NSEC_PER_USEC); + kbase_csf_firmware_set_gpu_idle_hysteresis_time(kbdev, (u64)dur_us * NSEC_PER_USEC); - return count; + return (ssize_t)count; } /** @@ -5416,7 +5528,7 @@ static ssize_t idle_hysteresis_time_store(struct device *dev, struct device_attr * @attr: The attributes of the sysfs file. * @buf: The output buffer to receive the GPU information. * - * This function is called to get the current idle hysteresis duration in ms. + * This function is called to get the current idle hysteresis duration in us. * * Return: The number of bytes output to @buf. */ @@ -5425,7 +5537,7 @@ static ssize_t idle_hysteresis_time_show(struct device *dev, struct device_attri { struct kbase_device *kbdev; ssize_t ret; - u32 dur; + u64 dur_us; CSTD_UNUSED(attr); @@ -5433,9 +5545,9 @@ static ssize_t idle_hysteresis_time_show(struct device *dev, struct device_attri if (!kbdev) return -ENODEV; - /* The unit of return value of idle_hysteresis_time_show is us, So divide by 1000.*/ - dur = kbase_csf_firmware_get_gpu_idle_hysteresis_time(kbdev) / NSEC_PER_USEC; - ret = scnprintf(buf, PAGE_SIZE, "%u\n", dur); + /* The unit of return value of idle_hysteresis_time_show is us, So divide by 1000 */ + dur_us = div_u64(kbase_csf_firmware_get_gpu_idle_hysteresis_time(kbdev), NSEC_PER_USEC); + ret = scnprintf(buf, PAGE_SIZE, "%u\n", (u32)dur_us); return ret; } @@ -5462,19 +5574,19 @@ static ssize_t idle_hysteresis_time_ns_store(struct device *dev, struct device_a const char *buf, size_t count) { struct kbase_device *kbdev; - u32 dur = 0; + u64 dur_ns = 0; kbdev = to_kbase_device(dev); if (!kbdev) return -ENODEV; - if (kstrtou32(buf, 0, &dur)) { + if (kstrtou64(buf, 0, &dur_ns)) { dev_err(kbdev->dev, "Couldn't process idle_hysteresis_time_ns write operation.\n" "Use format \n"); return -EINVAL; } - kbase_csf_firmware_set_gpu_idle_hysteresis_time(kbdev, dur); + kbase_csf_firmware_set_gpu_idle_hysteresis_time(kbdev, dur_ns); return count; } @@ -5496,14 +5608,14 @@ static ssize_t idle_hysteresis_time_ns_show(struct device *dev, struct device_at { struct kbase_device *kbdev; ssize_t ret; - u32 dur; + u64 dur_ns; kbdev = to_kbase_device(dev); if (!kbdev) return -ENODEV; - dur = kbase_csf_firmware_get_gpu_idle_hysteresis_time(kbdev); - ret = scnprintf(buf, PAGE_SIZE, "%u\n", dur); + dur_ns = kbase_csf_firmware_get_gpu_idle_hysteresis_time(kbdev); + ret = scnprintf(buf, PAGE_SIZE, "%llu\n", dur_ns); return ret; } @@ -5527,16 +5639,16 @@ static ssize_t mcu_shader_pwroff_timeout_show(struct device *dev, struct device_ char *const buf) { struct kbase_device *kbdev = dev_get_drvdata(dev); - u32 pwroff; + u64 pwroff_us; CSTD_UNUSED(attr); if (!kbdev) return -ENODEV; - /* The unit of return value of the function is us, So divide by 1000.*/ - pwroff = kbase_csf_firmware_get_mcu_core_pwroff_time(kbdev) / NSEC_PER_USEC; - return scnprintf(buf, PAGE_SIZE, "%u\n", pwroff); + /* The unit of return value of the function is us, So divide by 1000 */ + pwroff_us = div_u64(kbase_csf_firmware_get_mcu_core_pwroff_time(kbdev), NSEC_PER_USEC); + return scnprintf(buf, PAGE_SIZE, "%u\n", (u32)pwroff_us); } /** @@ -5557,7 +5669,7 @@ static ssize_t mcu_shader_pwroff_timeout_store(struct device *dev, struct device const char *buf, size_t count) { struct kbase_device *kbdev = dev_get_drvdata(dev); - u32 dur; + u32 dur_us; const struct kbase_pm_policy *current_policy; bool always_on; @@ -5567,20 +5679,20 @@ static ssize_t mcu_shader_pwroff_timeout_store(struct device *dev, struct device if (!kbdev) return -ENODEV; - if (kstrtouint(buf, 0, &dur)) + if (kstrtou32(buf, 0, &dur_us)) return -EINVAL; current_policy = kbase_pm_get_policy(kbdev); always_on = current_policy == &kbase_pm_always_on_policy_ops; - if (dur == 0 && !always_on) + if (dur_us == 0 && !always_on) return -EINVAL; /* In sysFs, The unit of the input value of mcu_shader_pwroff_timeout is us. * But the unit of the input parameter of this function is ns, so multiply by 1000 */ - kbase_csf_firmware_set_mcu_core_pwroff_time(kbdev, dur * NSEC_PER_USEC); + kbase_csf_firmware_set_mcu_core_pwroff_time(kbdev, (u64)dur_us * NSEC_PER_USEC); - return count; + return (ssize_t)count; } static DEVICE_ATTR_RW(mcu_shader_pwroff_timeout); @@ -5602,13 +5714,13 @@ static ssize_t mcu_shader_pwroff_timeout_ns_show(struct device *dev, struct devi char *const buf) { struct kbase_device *kbdev = dev_get_drvdata(dev); - u32 pwroff; + u64 pwroff_ns; if (!kbdev) return -ENODEV; - pwroff = kbase_csf_firmware_get_mcu_core_pwroff_time(kbdev); - return scnprintf(buf, PAGE_SIZE, "%u\n", pwroff); + pwroff_ns = kbase_csf_firmware_get_mcu_core_pwroff_time(kbdev); + return scnprintf(buf, PAGE_SIZE, "%llu\n", pwroff_ns); } /** @@ -5629,7 +5741,7 @@ static ssize_t mcu_shader_pwroff_timeout_ns_store(struct device *dev, struct dev const char *buf, size_t count) { struct kbase_device *kbdev = dev_get_drvdata(dev); - u32 dur; + u64 dur_ns; const struct kbase_pm_policy *current_policy; bool always_on; @@ -5637,15 +5749,15 @@ static ssize_t mcu_shader_pwroff_timeout_ns_store(struct device *dev, struct dev if (!kbdev) return -ENODEV; - if (kstrtouint(buf, 0, &dur)) + if (kstrtou64(buf, 0, &dur_ns)) return -EINVAL; current_policy = kbase_pm_get_policy(kbdev); always_on = current_policy == &kbase_pm_always_on_policy_ops; - if (dur == 0 && !always_on) + if (dur_ns == 0 && !always_on) return -EINVAL; - kbase_csf_firmware_set_mcu_core_pwroff_time(kbdev, dur); + kbase_csf_firmware_set_mcu_core_pwroff_time(kbdev, dur_ns); return count; } @@ -5680,6 +5792,8 @@ static struct attribute *kbase_attrs[] = { &dev_attr_js_scheduling_period.attr, #else &dev_attr_csg_scheduling_period.attr, + &dev_attr_add_prioritized_process.attr, + &dev_attr_remove_prioritized_process.attr, &dev_attr_fw_timeout.attr, &dev_attr_idle_hysteresis_time.attr, &dev_attr_idle_hysteresis_time_ns.attr, @@ -5833,6 +5947,10 @@ static int kbase_platform_device_probe(struct platform_device *pdev) #endif dev_info(kbdev->dev, "Probed as %s\n", dev_name(kbdev->mdev.this_device)); + if (PAGE_SHIFT != 12) + dev_warn(kbdev->dev, "Experimental feature: %s with Page Size of %luKiB", + dev_name(kbdev->mdev.this_device), PAGE_SIZE / 1024); + kbase_increment_device_id(); #if (KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) mutex_unlock(&kbase_probe_mutex); @@ -6150,8 +6268,8 @@ void kbase_trace_mali_pm_status(u32 dev_id, u32 event, u64 value) void kbase_trace_mali_job_slots_event(u32 dev_id, u32 event, const struct kbase_context *kctx, u8 atom_id) { - trace_mali_job_slots_event(dev_id, event, (kctx != NULL ? kctx->tgid : 0), - (kctx != NULL ? kctx->pid : 0), atom_id); + trace_mali_job_slots_event(dev_id, event, (kctx != NULL ? (u32)kctx->tgid : 0U), + (kctx != NULL ? (u32)kctx->pid : 0U), atom_id); } void kbase_trace_mali_page_fault_insert_pages(u32 dev_id, int event, u32 value) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c index 871d7d0b8395..41f8c9ca2bf8 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c @@ -45,7 +45,7 @@ static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx) int kbase_ctx_sched_init(struct kbase_device *kbdev) { - int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; + int as_present = (1 << kbdev->nr_hw_address_spaces) - 1; /* These two must be recalculated if nr_hw_address_spaces changes * (e.g. for HW workarounds) @@ -133,7 +133,8 @@ int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx) } kctx->as_nr = free_as; kbdev->as_to_kctx[free_as] = kctx; - KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS(kbdev, kctx->id, free_as); + KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS(kbdev, kctx->id, + (u32)free_as); kbase_mmu_update(kbdev, &kctx->mmu, kctx->as_nr); } } else { @@ -198,7 +199,7 @@ void kbase_ctx_sched_release_ctx(struct kbase_context *kctx) } } - KBASE_KTRACE_ADD(kbdev, SCHED_RELEASE_CTX, kctx, new_ref_count); + KBASE_KTRACE_ADD(kbdev, SCHED_RELEASE_CTX, kctx, (u64)new_ref_count); } void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx) @@ -340,7 +341,7 @@ bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx) kbase_ctx_sched_retain_ctx_refcount(kctx); KBASE_KTRACE_ADD(kctx->kbdev, SCHED_RETAIN_CTX_NOLOCK, kctx, - kbase_ktrace_get_ctx_refcnt(kctx)); + (u64)kbase_ktrace_get_ctx_refcnt(kctx)); result = true; } @@ -408,7 +409,7 @@ bool kbase_ctx_sched_inc_refcount_if_as_valid(struct kbase_context *kctx) kbdev->as_free &= ~(1u << kctx->as_nr); KBASE_KTRACE_ADD(kbdev, SCHED_RETAIN_CTX_NOLOCK, kctx, - kbase_ktrace_get_ctx_refcnt(kctx)); + (u64)kbase_ktrace_get_ctx_refcnt(kctx)); added_ref = true; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h index 397724267fdf..fd1b82471d26 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h @@ -22,7 +22,10 @@ #ifndef _KBASE_CTX_SCHED_H_ #define _KBASE_CTX_SCHED_H_ -#include +#include + +struct kbase_context; +struct kbase_device; /** * DOC: The Context Scheduler manages address space assignment and reference @@ -60,7 +63,7 @@ int kbase_ctx_sched_init(struct kbase_device *kbdev); void kbase_ctx_sched_term(struct kbase_device *kbdev); /** - * kbase_ctx_sched_ctx_init - Initialize per-context data fields for scheduling + * kbase_ctx_sched_init_ctx - Initialize per-context data fields for scheduling * @kctx: The context to initialize * * This must be called during context initialization before any other context diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug.c b/drivers/gpu/arm/bifrost/mali_kbase_debug.c index 77442feb1fda..680ba0184215 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_debug.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug.c @@ -19,7 +19,8 @@ * */ -#include +#include +#include static struct kbasep_debug_assert_cb kbasep_debug_assert_registered_cb = { NULL, NULL }; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug.h b/drivers/gpu/arm/bifrost/mali_kbase_debug.h index 876ecdd5c617..d4edb1d6f0e9 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_debug.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug.h @@ -22,7 +22,10 @@ #ifndef _KBASE_DEBUG_H #define _KBASE_DEBUG_H +#include + #include +#include /** @brief If equals to 0, a trace containing the file, line, and function will be displayed before each message. */ #define KBASE_DEBUG_SKIP_TRACE 0 diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h b/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h index 0d7d2be28575..ee1228155621 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_job_fault.h @@ -22,8 +22,10 @@ #ifndef _KBASE_DEBUG_JOB_FAULT_H #define _KBASE_DEBUG_JOB_FAULT_H -#include -#include +#include + +struct kbase_context; +struct kbase_device; #define REGISTER_DUMP_TERMINATION_FLAG 0xFFFFFFFF diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.h b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.h index 8cf69c2cbaf9..28df0eaba0bf 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,7 +22,7 @@ #ifndef _KBASE_DEBUG_MEM_ALLOCS_H #define _KBASE_DEBUG_MEM_ALLOCS_H -#include +struct kbase_context; /** * kbase_debug_mem_allocs_init() - Initialize the mem_allocs debugfs file diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c index eb587bd7f6e9..dd8f8ff6fe79 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c @@ -83,7 +83,7 @@ static void *debug_mem_start(struct seq_file *m, loff_t *_pos) if (!data) return NULL; data->lh = &map->node; - data->offset = pos; + data->offset = (size_t)pos; return data; } } @@ -360,7 +360,7 @@ static ssize_t debug_mem_write(struct file *file, const char __user *ubuf, size_ kctx->mem_view_column_width = column_width; kbase_gpu_vm_unlock(kctx); - return count; + return (ssize_t)count; } static const struct file_operations kbase_debug_mem_view_fops = { .owner = THIS_MODULE, diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_zones.h b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_zones.h index acf349b608d5..275c863ec159 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_zones.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_zones.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,7 +22,7 @@ #ifndef _KBASE_DEBUG_MEM_ZONES_H #define _KBASE_DEBUG_MEM_ZONES_H -#include +struct kbase_context; /** * kbase_debug_mem_zones_init() - Initialize the mem_zones sysfs file diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c b/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c index 69e715caf7cc..0686f32ea08d 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_debugfs_helper.c @@ -48,7 +48,8 @@ static int set_attr_from_string(char *const buf, void *const array, size_t const nelems, kbase_debugfs_helper_set_attr_fn *const set_attr_fn) { - size_t index, err = 0; + size_t index; + int err = 0; char *ptr = buf; for (index = 0; index < nelems && *ptr; ++index) { @@ -175,8 +176,8 @@ ssize_t kbase_debugfs_helper_get_attr_to_string(char *const buf, size_t const si if (index == (nelems - 1)) postfix = "\n"; - total += scnprintf(buf + total, size - total, "%zu%s", get_attr_fn(array, index), - postfix); + total += scnprintf(buf + total, size - (size_t)total, "%zu%s", + get_attr_fn(array, index), postfix); } return total; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_defs.h index 9fbcde665250..13a5c30dcb61 100755 --- a/drivers/gpu/arm/bifrost/mali_kbase_defs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_defs.h @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -45,13 +46,9 @@ #include #endif -#include +#include "debug/mali_kbase_debug_ktrace_defs.h" -#include -#include -#include -#include -#include +#include #include @@ -71,12 +68,17 @@ #include #endif /* CONFIG_MALI_ARBITER_SUPPORT */ -#include -#include #include #include -#include "debug/mali_kbase_debug_ktrace_defs.h" +#include +#include +#include +#include +#include +#include +#include +#include /** Number of milliseconds before we time out on a GPU soft/hard reset */ #define RESET_TIMEOUT 500 @@ -522,8 +524,11 @@ struct kbase_pm_device_data { * @cur_size: Number of free pages currently in the pool (may exceed * @max_size in some corner cases) * @max_size: Maximum number of free pages in the pool - * @order: order = 0 refers to a pool of 4 KB pages - * order = 9 refers to a pool of 2 MB pages (2^9 * 4KB = 2 MB) + * @order: order = 0 refers to a pool of small pages + * order != 0 refers to a pool of 2 MB pages, so + * order = 9 (when small page size is 4KB, 2^9 * 4KB = 2 MB) + * order = 7 (when small page size is 16KB, 2^7 * 16KB = 2 MB) + * order = 5 (when small page size is 64KB, 2^5 * 64KB = 2 MB) * @group_id: A memory group ID to be passed to a platform-specific * memory group manager, if present. Immutable. * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). @@ -562,14 +567,14 @@ struct kbase_mem_pool { /** * struct kbase_mem_pool_group - a complete set of physical memory pools. * - * @small: Array of objects containing the state for pools of 4 KiB size + * @small: Array of objects containing the state for pools of small size * physical pages. * @large: Array of objects containing the state for pools of 2 MiB size * physical pages. * * Memory pools are used to allow efficient reallocation of previously-freed * physical pages. A pair of memory pools is initialized for each physical - * memory group: one for 4 KiB pages and one for 2 MiB pages. These arrays + * memory group: one for small pages and one for 2 MiB pages. These arrays * should be indexed by physical memory group ID, the meaning of which is * defined by the systems integrator. */ @@ -592,7 +597,7 @@ struct kbase_mem_pool_config { * struct kbase_mem_pool_group_config - Initial configuration for a complete * set of physical memory pools * - * @small: Array of initial configuration for pools of 4 KiB pages. + * @small: Array of initial configuration for pools of small pages. * @large: Array of initial configuration for pools of 2 MiB pages. * * This array should be indexed by physical memory group ID, the meaning @@ -777,6 +782,7 @@ struct kbase_mem_migrate { * power management, cache etc.) * @irqs.irq: irq number * @irqs.flags: irq flags + * @nr_irqs: The number of interrupt entries. * @clocks: Pointer to the input clock resources referenced by * the GPU device node. * @scmi_clk: Pointer to the input scmi clock resources @@ -832,8 +838,8 @@ struct kbase_mem_migrate { * group manager if no platform-specific memory group * manager was retrieved through device tree. * @mmu_unresponsive: Flag to indicate MMU is not responding. - * Set if a MMU command isn't completed within - * &kbase_device:mmu_or_gpu_cache_op_wait_time_ms. + * Set if a MMU command isn't completed within the + * MMU_AS_INACTIVE_WAIT_TIMEOUT scaled timeout. * Clear by kbase_ctx_sched_restore_all_as() after GPU reset completes. * @as: Array of objects representing address spaces of GPU. * @as_to_kctx: Array of pointers to struct kbase_context, having @@ -987,13 +993,6 @@ struct kbase_mem_migrate { * backend specific data for HW access layer. * @faults_pending: Count of page/bus faults waiting for bottom half processing * via workqueues. - * @mmu_hw_operation_in_progress: Set before sending the MMU command and is - * cleared after the command is complete. Whilst this - * flag is set, the write to L2_PWROFF register will be - * skipped which is needed to workaround the HW issue - * GPU2019-3878. PM state machine is invoked after - * clearing this flag and @hwaccess_lock is used to - * serialize the access. * @mmu_page_migrate_in_progress: Set before starting a MMU page migration transaction * and cleared after the transaction completes. PM L2 state is * prevented from entering powering up/down transitions when the @@ -1082,11 +1081,12 @@ struct kbase_mem_migrate { * KCPU queue. These structures may outlive kbase module * itself. Therefore, in such a case, a warning should be * be produced. - * @mmu_or_gpu_cache_op_wait_time_ms: Maximum waiting time in ms for the completion of - * a cache operation via MMU_AS_CONTROL or GPU_CONTROL. * @va_region_slab: kmem_cache (slab) for allocated kbase_va_region structures. * @fence_signal_timeout_enabled: Global flag for whether fence signal timeout tracking * is enabled. + * @pcm_prioritized_process_nb: Notifier block for the Priority Control Manager + * driver, this is used to be informed of the + * changes in the list of prioritized processes. */ struct kbase_device { u32 hw_quirks_sc; @@ -1106,9 +1106,10 @@ struct kbase_device { size_t size; } regmap; struct { - int irq; - int flags; + u32 irq; + u32 flags; } irqs[3]; + u32 nr_irqs; struct clk *clocks[BASE_MAX_NR_CLOCKS_REGULATORS]; unsigned int nr_clocks; @@ -1289,9 +1290,6 @@ struct kbase_device { atomic_t faults_pending; -#if MALI_USE_CSF - bool mmu_hw_operation_in_progress; -#endif bool mmu_page_migrate_in_progress; bool poweroff_pending; @@ -1366,7 +1364,7 @@ struct kbase_device { struct { struct kbase_context *kctx; u64 jc; - int slot; + u32 slot; u64 flags; } dummy_job_wa; bool dummy_job_wa_loaded; @@ -1397,7 +1395,6 @@ struct kbase_device { #if MALI_USE_CSF && IS_ENABLED(CONFIG_SYNC_FILE) atomic_t live_fence_metadata; #endif - u32 mmu_or_gpu_cache_op_wait_time_ms; struct kmem_cache *va_region_slab; #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) @@ -1409,6 +1406,8 @@ struct kbase_device { #if MALI_USE_CSF atomic_t fence_signal_timeout_enabled; #endif + + struct notifier_block pcm_prioritized_process_nb; }; /** @@ -1560,6 +1559,11 @@ struct kbase_file { * @KCTX_JPL_ENABLED: Set when JIT physical page limit is less than JIT virtual * address page limit, so we must take care to not exceed the physical limit * + * @KCTX_PAGE_FAULT_REPORT_SKIP: Set when the GPU page fault handler is not + * allowed to allocate a physical page due to the process exit or context + * termination. It is used to suppress the error messages that ensue because + * the page fault didn't get handled. + * * All members need to be separate bits. This enum is intended for use in a * bitmask where multiple values get OR-ed together. */ @@ -1580,6 +1584,7 @@ enum kbase_context_flags { KCTX_PULLED_SINCE_ACTIVE_JS2 = 1U << 14, KCTX_AS_DISABLED_ON_FAULT = 1U << 15, KCTX_JPL_ENABLED = 1U << 16, + KCTX_PAGE_FAULT_REPORT_SKIP = 1U << 17, }; #else /** @@ -1638,6 +1643,11 @@ enum kbase_context_flags { * refcount for the context drops to 0 or on when the address spaces are * re-enabled on GPU reset or power cycle. * + * @KCTX_PAGE_FAULT_REPORT_SKIP: Set when the GPU page fault handler is not + * allowed to allocate a physical page due to the process exit or context + * termination. It is used to suppress the error messages that ensue because + * the page fault didn't get handled. + * * All members need to be separate bits. This enum is intended for use in a * bitmask where multiple values get OR-ed together. */ @@ -1657,13 +1667,14 @@ enum kbase_context_flags { KCTX_PULLED_SINCE_ACTIVE_JS1 = 1U << 13, KCTX_PULLED_SINCE_ACTIVE_JS2 = 1U << 14, KCTX_AS_DISABLED_ON_FAULT = 1U << 15, + KCTX_PAGE_FAULT_REPORT_SKIP = 1U << 16, }; #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ struct kbase_sub_alloc { struct list_head link; struct page *page; - DECLARE_BITMAP(sub_pages, SZ_2M / SZ_4K); + DECLARE_BITMAP(sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE); }; /** @@ -1703,7 +1714,7 @@ struct kbase_sub_alloc { * @mem_partials_lock: Lock for protecting the operations done on the elements * added to @mem_partials list. * @mem_partials: List head for the list of large pages, 2MB in size, which - * have been split into 4 KB pages and are used partially + * have been split into small pages and are used partially * for the allocations >= 2 MB in size. * @reg_lock: Lock used for GPU virtual address space management operations, * like adding/freeing a memory region in the address space. @@ -1733,10 +1744,17 @@ struct kbase_sub_alloc { * which actually created the context. This is usually, * but not necessarily, the same as the thread which * opened the device file /dev/malixx instance. + * @prioritized: Indicate whether work items originating from this + * context should be treated with a higher priority + * level relative to work items with the same priority + * from other contexts. This value could change multiple + * times over the life time of the context, such as when + * an application becomes foreground or goes to the + * background. * @csf: kbase csf context * @jctx: object encapsulating all the Job dispatcher related state, * including the array of atoms. - * @used_pages: Keeps a track of the number of 4KB physical pages in use + * @used_pages: Keeps a track of the number of small physical pages in use * for the context. * @nonmapped_pages: Updated in the same way as @used_pages, except for the case * when special tracking page is freed by userspace where it @@ -1791,7 +1809,7 @@ struct kbase_sub_alloc { * on this descriptor for the Userspace created contexts so that * Kbase can safely access it to update the memory usage counters. * The reference is dropped on context termination. - * @gpu_va_end: End address of the GPU va space (in 4KB page units) + * @gpu_va_end: End address of the GPU va space (in small page units) * @running_total_tiler_heap_nr_chunks: Running total of number of chunks in all * tiler heaps of the kbase context. * @running_total_tiler_heap_memory: Running total of the tiler heap memory in the @@ -1981,6 +1999,7 @@ struct kbase_context { pid_t tgid; pid_t pid; + atomic_t prioritized; atomic_t used_pages; atomic_t nonmapped_pages; atomic_t permanent_mapped_pages; @@ -2166,10 +2185,4 @@ static inline u64 kbase_get_lock_region_min_size_log2(struct kbase_gpu_props con #define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x)) * 1000000U)) #define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x)) -/* Maximum number of loops polling the GPU for a cache flush before we assume it must have completed */ -#define KBASE_CLEAN_CACHE_MAX_LOOPS 100000 -/* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */ -#define KBASE_AS_INACTIVE_MAX_LOOPS 100000000 -/* Maximum number of loops polling the GPU PRFCNT_ACTIVE bit before we assume the GPU has hung */ -#define KBASE_PRFCNT_ACTIVE_MAX_LOOPS 100000000 #endif /* _KBASE_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c b/drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c index dc2df465f3b2..305b38c343e6 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c @@ -74,6 +74,6 @@ u32 kbase_disjoint_event_get(struct kbase_device *kbdev) { KBASE_DEBUG_ASSERT(kbdev != NULL); - return atomic_read(&kbdev->disjoint_event.count); + return (u32)atomic_read(&kbdev->disjoint_event.count); } KBASE_EXPORT_TEST_API(kbase_disjoint_event_get); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c index 4b322e62f2de..9c39f0e20f76 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c @@ -53,7 +53,7 @@ struct wa_blob { u32 blob_offset; } __packed; -static bool in_range(const u8 *base, const u8 *end, off_t off, size_t sz) +static bool within_range(const u8 *base, const u8 *end, off_t off, size_t sz) { return !((size_t)(end - base - off) < sz); } @@ -80,39 +80,7 @@ static u32 wait_any(struct kbase_device *kbdev, off_t offset, u32 bits) return (val & bits); } -static int wait(struct kbase_device *kbdev, off_t offset, u64 bits, bool set) -{ - int loop; - const int timeout = 100; - u64 val; - u64 target = 0; - - if (set) - target = bits; - - for (loop = 0; loop < timeout; loop++) { - if (kbase_reg_is_size64(kbdev, offset)) - val = kbase_reg_read64(kbdev, offset); - else - val = kbase_reg_read32(kbdev, offset); - - if ((val & bits) == target) - break; - - udelay(10); - } - - if (loop == timeout) { - dev_err(kbdev->dev, - "Timeout reading register 0x%lx, bits 0x%llx, last read was 0x%llx\n", - (unsigned long)offset, bits, val); - return -ETIMEDOUT; - } - - return 0; -} - -static inline int run_job(struct kbase_device *kbdev, int as, int slot, u64 cores, u64 jc) +static inline int run_job(struct kbase_device *kbdev, int as, u32 slot, u64 cores, u64 jc) { u32 done; @@ -120,7 +88,7 @@ static inline int run_job(struct kbase_device *kbdev, int as, int slot, u64 core kbase_reg_write64(kbdev, JOB_SLOT_OFFSET(slot, HEAD_NEXT), jc); kbase_reg_write64(kbdev, JOB_SLOT_OFFSET(slot, AFFINITY_NEXT), cores); kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(slot, CONFIG_NEXT), - JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK | as); + JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK | (unsigned int)as); /* go */ kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(slot, COMMAND_NEXT), JS_COMMAND_START); @@ -131,7 +99,7 @@ static inline int run_job(struct kbase_device *kbdev, int as, int slot, u64 core kbase_reg_write32(kbdev, JOB_CONTROL_ENUM(JOB_IRQ_CLEAR), done); if (done != (1ul << slot)) { - dev_err(kbdev->dev, "Failed to run WA job on slot %d cores 0x%llx: done 0x%lx\n", + dev_err(kbdev->dev, "Failed to run WA job on slot %u cores 0x%llx: done 0x%lx\n", slot, (unsigned long long)cores, (unsigned long)done); dev_err(kbdev->dev, "JS_STATUS on failure: 0x%x\n", kbase_reg_read32(kbdev, JOB_SLOT_OFFSET(slot, STATUS))); @@ -146,12 +114,14 @@ static inline int run_job(struct kbase_device *kbdev, int as, int slot, u64 core int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores) { int as; - int slot; + u32 slot; u64 jc; int failed = 0; int runs = 0; u32 old_gpu_mask; u32 old_job_mask; + u64 val; + const u32 timeout_us = 10000; if (!kbdev) return -EFAULT; @@ -174,7 +144,8 @@ int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores) if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP) { /* wait for power-ups */ - wait(kbdev, GPU_CONTROL_ENUM(SHADER_READY), cores, true); + kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(SHADER_READY), val, + (val & cores) == cores, 10, timeout_us, false); } if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE) { @@ -205,8 +176,10 @@ int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores) kbase_reg_write64(kbdev, GPU_CONTROL_ENUM(SHADER_PWROFF), cores); /* wait for power off complete */ - wait(kbdev, GPU_CONTROL_ENUM(SHADER_READY), cores, false); - wait(kbdev, GPU_CONTROL_ENUM(SHADER_PWRTRANS), cores, false); + kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(SHADER_READY), val, !(val & cores), + 10, timeout_us, false); + kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(SHADER_PWRTRANS), val, + !(val & cores), 10, timeout_us, false); kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_CLEAR), U32_MAX); } @@ -297,7 +270,7 @@ int kbase_dummy_job_wa_load(struct kbase_device *kbdev) dev_dbg(kbdev->dev, "Loaded firmware of size %zu bytes\n", firmware->size); - if (!in_range(fw, fw_end, 0, sizeof(*header))) { + if (!within_range(fw, fw_end, 0, sizeof(*header))) { dev_err(kbdev->dev, "WA too small\n"); goto bad_fw; } @@ -316,7 +289,7 @@ int kbase_dummy_job_wa_load(struct kbase_device *kbdev) goto bad_fw; } - if (!in_range(fw, fw_end, header->info_offset, sizeof(*v2_info))) { + if (!within_range(fw, fw_end, header->info_offset, sizeof(*v2_info))) { dev_err(kbdev->dev, "WA info offset out of bounds\n"); goto bad_fw; } @@ -342,14 +315,14 @@ int kbase_dummy_job_wa_load(struct kbase_device *kbdev) u64 gpu_va; struct kbase_va_region *va_region; - if (!in_range(fw, fw_end, blob_offset, sizeof(*blob))) { + if (!within_range(fw, fw_end, blob_offset, sizeof(*blob))) { dev_err(kbdev->dev, "Blob offset out-of-range: 0x%lx\n", (unsigned long)blob_offset); goto bad_fw; } blob = (const struct wa_blob *)(fw + blob_offset); - if (!in_range(fw, fw_end, blob->payload_offset, blob->size)) { + if (!within_range(fw, fw_end, blob->payload_offset, blob->size)) { dev_err(kbdev->dev, "Payload out-of-bounds\n"); goto bad_fw; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence.c b/drivers/gpu/arm/bifrost/mali_kbase_fence.c index 023bc6715224..febf2fd5643c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_fence.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_fence.c @@ -25,20 +25,14 @@ #include #include +#include + /* Spin lock protecting all Mali fences as fence->lock. */ static DEFINE_SPINLOCK(kbase_fence_lock); -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom) -#else struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom) -#endif { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; -#else struct dma_fence *fence; -#endif WARN_ON(katom->dma_fence.fence); @@ -47,7 +41,7 @@ struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom) return NULL; dma_fence_init(fence, &kbase_fence_ops, &kbase_fence_lock, katom->dma_fence.context, - atomic_inc_return(&katom->dma_fence.seqno)); + (u64)atomic_inc_return(&katom->dma_fence.seqno)); katom->dma_fence.fence = fence; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence.h b/drivers/gpu/arm/bifrost/mali_kbase_fence.h index f170e95b39d9..06690d4f17bb 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_fence.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_fence.h @@ -29,9 +29,9 @@ #if IS_ENABLED(CONFIG_SYNC_FILE) -#include #include "mali_kbase.h" -#include "mali_kbase_refcount_defs.h" + +#include #include #if MALI_USE_CSF @@ -52,7 +52,7 @@ struct kbase_kcpu_dma_fence_meta { kbase_refcount_t refcount; struct kbase_device *kbdev; - int kctx_id; + u32 kctx_id; char timeline_name[MAX_TIMELINE_NAME]; }; @@ -64,20 +64,12 @@ struct kbase_kcpu_dma_fence_meta { * @metadata: Pointer to metadata structure. */ struct kbase_kcpu_dma_fence { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence base; -#else struct dma_fence base; -#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ struct kbase_kcpu_dma_fence_meta *metadata; }; #endif -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -extern const struct fence_ops kbase_fence_ops; -#else extern const struct dma_fence_ops kbase_fence_ops; -#endif /** * kbase_fence_out_new() - Creates a new output fence and puts it on the atom @@ -85,11 +77,7 @@ extern const struct dma_fence_ops kbase_fence_ops; * * Return: A new fence object on success, NULL on failure. */ -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); -#else struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); -#endif #if IS_ENABLED(CONFIG_SYNC_FILE) /** @@ -197,11 +185,7 @@ static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, int status #define kbase_fence_get(fence_info) dma_fence_get((fence_info)->fence) #if MALI_USE_CSF -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -static inline struct kbase_kcpu_dma_fence *kbase_kcpu_dma_fence_get(struct fence *fence) -#else static inline struct kbase_kcpu_dma_fence *kbase_kcpu_dma_fence_get(struct dma_fence *fence) -#endif { if (fence->ops == &kbase_fence_ops) return (struct kbase_kcpu_dma_fence *)fence; @@ -217,11 +201,7 @@ static inline void kbase_kcpu_dma_fence_meta_put(struct kbase_kcpu_dma_fence_met } } -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -static inline void kbase_kcpu_dma_fence_put(struct fence *fence) -#else static inline void kbase_kcpu_dma_fence_put(struct dma_fence *fence) -#endif { struct kbase_kcpu_dma_fence *kcpu_fence = kbase_kcpu_dma_fence_get(fence); @@ -234,11 +214,7 @@ static inline void kbase_kcpu_dma_fence_put(struct dma_fence *fence) * kbase_fence_put() - Releases a reference to a fence * @fence: Fence to release reference for. */ -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -static inline void kbase_fence_put(struct fence *fence) -#else static inline void kbase_fence_put(struct dma_fence *fence) -#endif { dma_fence_put(fence); } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c b/drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c index 7315da8b89f9..b10e69cea565 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_fence_ops.c @@ -63,7 +63,7 @@ static void kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int else format = "%llu"; - if (unlikely(!scnprintf(str, size, format, fence->seqno))) + if (unlikely(!scnprintf(str, (size_t)size, format, fence->seqno))) pr_err("Fail to encode fence seqno to string"); } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h index 6ad773658554..6993693cd8da 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.h @@ -27,8 +27,11 @@ #ifndef _KBASE_GPU_MEMORY_DEBUGFS_H #define _KBASE_GPU_MEMORY_DEBUGFS_H -#include -#include +#include +#include + +struct kbase_io_history; +struct kbase_device; /* kbase_io_history_add - add new entry to the register access history * diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c index 8e00c0dc830e..3a5b97db7c04 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c @@ -20,9 +20,14 @@ */ #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) + #include "mali_power_gpu_work_period_trace.h" #include #include +#include + +#include +#include /** * enum gpu_metrics_ctx_flags - Flags for the GPU metrics context diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.h b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.h index c89e25996f52..c445dff32dc9 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.h @@ -27,7 +27,12 @@ #define _KBASE_GPU_METRICS_H_ #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) -#include + +#include + +struct kbase_device; +struct kbase_context; +struct kbase_gpu_metrics_ctx; /** * kbase_gpu_metrics_get_tp_emit_interval() - Return the trace point emission interval. diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c index 190800394292..10b3b506e84e 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c @@ -326,6 +326,8 @@ void kbase_gpuprops_term(struct kbase_device *kbdev) static u8 override_l2_size; module_param(override_l2_size, byte, 0000); MODULE_PARM_DESC(override_l2_size, "Override L2 size config for testing"); +/* Minimum L2 cache size - LOG2(1KiB) */ +#define OVERRIDE_L2_SIZE_MIN_LOG2 (10) static u8 override_l2_hash; module_param(override_l2_hash, byte, 0000); @@ -376,6 +378,9 @@ static enum l2_config_override_result kbase_read_l2_config_from_dt(struct kbase_ else if (of_property_read_u8(np, "l2-size", &kbdev->l2_size_override)) kbdev->l2_size_override = 0; + if (kbdev->l2_size_override != 0 && kbdev->l2_size_override < OVERRIDE_L2_SIZE_MIN_LOG2) + return L2_CONFIG_OVERRIDE_FAIL; + /* Check overriding value is supported, if not will result in * undefined behavior. */ @@ -620,7 +625,6 @@ static void kbase_populate_user_data(struct kbase_device *kbdev, struct gpu_prop data->raw_props.coherency_mode = kprops->coherency_mode; /* Properties (mostly) from raw register values */ - /* For compatibility, we are passing the lower 32-bits of the gpu_id */ data->raw_props.gpu_id = regdump->gpu_id; { @@ -649,7 +653,7 @@ static void kbase_populate_user_data(struct kbase_device *kbdev, struct gpu_prop data->l2_props.log2_cache_size = KBASE_UBFX64(regdump->l2_features, 16U, 8); data->coherency_info.coherency = regdump->mem_features; - data->tiler_props.bin_size_bytes = 1 << KBASE_UBFX64(regdump->tiler_features, 0U, 6); + data->tiler_props.bin_size_bytes = 1U << KBASE_UBFX64(regdump->tiler_features, 0U, 6); data->tiler_props.max_active_levels = KBASE_UBFX32(regdump->tiler_features, 8U, 4); if (regdump->thread_max_workgroup_size == 0) @@ -728,7 +732,7 @@ int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev) for (i = 0; i < count; i++) { /* 4 bytes for the ID, and the size of the property */ - size += 4 + gpu_property_mapping[i].size; + size += (u32)(4 + gpu_property_mapping[i].size); } kprops->prop_buffer_size = size; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h index 093f9680ece6..46857858c80d 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h @@ -26,10 +26,13 @@ #ifndef _KBASE_GPUPROPS_H_ #define _KBASE_GPUPROPS_H_ -#include "mali_kbase_gpuprops_types.h" +#include /* Forward definition - see mali_kbase.h */ struct kbase_device; +struct max_config_props; +struct curr_config_props; +struct kbase_gpu_id_props; /** * KBASE_UBFX32 - Extracts bits from a 32-bit bitfield. diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gwt.c b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c index 79c3d5129324..c92d54c9e663 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gwt.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c @@ -19,8 +19,15 @@ * */ -#include "mali_kbase_gwt.h" +#include +#include +#include +#include +#include + +#include #include +#include static inline void kbase_gpu_gwt_setup_page_permission(struct kbase_context *kctx, unsigned long flag, struct rb_node *node) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gwt.h b/drivers/gpu/arm/bifrost/mali_kbase_gwt.h index e184375d0589..ee7f3bf6b0ac 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gwt.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_gwt.h @@ -22,9 +22,10 @@ #if !defined(_KBASE_GWT_H) #define _KBASE_GWT_H -#include #include +struct kbase_context; + /** * kbase_gpu_gwt_start - Start the GPU write tracking * @kctx: Pointer to kernel context diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hw.c b/drivers/gpu/arm/bifrost/mali_kbase_hw.c index dd0873f03125..7d4200e96fd3 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hw.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_hw.c @@ -205,6 +205,8 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id(struct kbase_dev { { GPU_ID_VERSION_MAKE(0, 0, 0), base_hw_issues_tBAx_r0p0 }, { GPU_ID_VERSION_MAKE(0, 0, 1), base_hw_issues_tBAx_r0p0 }, { GPU_ID_VERSION_MAKE(0, 0, 2), base_hw_issues_tBAx_r0p0 }, + { GPU_ID_VERSION_MAKE(0, 1, 0), base_hw_issues_tBAx_r0p1 }, + { GPU_ID_VERSION_MAKE(0, 2, 0), base_hw_issues_tBAx_r0p2 }, { U32_MAX, NULL } } }, { GPU_ID_PRODUCT_TODX, @@ -244,17 +246,21 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id(struct kbase_dev { GPU_ID_PRODUCT_TTIX, { { GPU_ID_VERSION_MAKE(0, 0, 0), base_hw_issues_tTIx_r0p0 }, + { GPU_ID_VERSION_MAKE(0, 1, 0), base_hw_issues_tTIx_r0p1 }, { U32_MAX, NULL } } }, { GPU_ID_PRODUCT_LTIX, { { GPU_ID_VERSION_MAKE(0, 0, 0), base_hw_issues_tTIx_r0p0 }, + { GPU_ID_VERSION_MAKE(0, 1, 0), base_hw_issues_tTIx_r0p1 }, { U32_MAX, NULL } } }, { GPU_ID_PRODUCT_TKRX, { { GPU_ID_VERSION_MAKE(0, 0, 0), base_hw_issues_tKRx_r0p0 }, + { GPU_ID_VERSION_MAKE(0, 1, 0), base_hw_issues_tKRx_r0p1 }, { U32_MAX, NULL } } }, { GPU_ID_PRODUCT_LKRX, { { GPU_ID_VERSION_MAKE(0, 0, 0), base_hw_issues_tKRx_r0p0 }, + { GPU_ID_VERSION_MAKE(0, 1, 0), base_hw_issues_tKRx_r0p1 }, { U32_MAX, NULL } } }, }; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h index 62a6ec51b17f..96aeb6188a05 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014, 2016-2018, 2020-2021 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -26,7 +26,7 @@ #ifndef _KBASE_HWACCESS_DEFS_H_ #define _KBASE_HWACCESS_DEFS_H_ -#include +#include /** * struct kbase_hwaccess_data - object encapsulating the GPU backend specific diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h index 93003754820d..ed91019ff74c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h @@ -194,7 +194,7 @@ int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, unsigned int js); * * Return: Number of atoms currently on slot @js that are currently on the GPU. */ -int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, unsigned int js); +u32 kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, unsigned int js); /** * kbase_backend_ctx_count_changed() - Number of contexts ready to submit jobs diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h index fc43a3fcd69f..7a0ea49099ba 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_pm.h @@ -26,13 +26,13 @@ #ifndef _KBASE_HWACCESS_PM_H_ #define _KBASE_HWACCESS_PM_H_ -#include +#include #include -#include - /* Forward definition - see mali_kbase.h */ struct kbase_device; +struct kbase_pm_policy; +struct kbase_pm_ca_policy; /* Functions common to all HW access backends */ @@ -199,7 +199,8 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic * * Return: The number of policies */ -int kbase_pm_list_policies(struct kbase_device *kbdev, const struct kbase_pm_policy *const **list); +size_t kbase_pm_list_policies(struct kbase_device *kbdev, + const struct kbase_pm_policy *const **list); /** * kbase_pm_protected_mode_enable() - Enable protected mode diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h index 8e5a8137be45..0630dfa6db3a 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h @@ -149,6 +149,15 @@ unsigned int kbase_get_timeout_ms(struct kbase_device *kbdev, enum kbase_timeout */ u64 kbase_backend_get_cycle_cnt(struct kbase_device *kbdev); +/** + * kbase_arch_timer_get_cntfrq - Get system timestamp counter frequency. + * + * @kbdev: Instance of a GPU platform device. + * + * Return: Frequency in Hz + */ +u64 kbase_arch_timer_get_cntfrq(struct kbase_device *kbdev); + /** * kbase_backend_time_init() - Initialize system timestamp converter. * diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd.c b/drivers/gpu/arm/bifrost/mali_kbase_jd.c index f66529485975..418a1913b241 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_jd.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_jd.c @@ -224,7 +224,8 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, struct kbase_va_region *reg; reg = kbase_region_tracker_find_region_enclosing_address( - katom->kctx, user_res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE); + katom->kctx, + user_res->ext_resource & ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE); /* did we find a matching region object? */ if (unlikely(kbase_is_region_invalid_or_free(reg))) { /* roll back */ @@ -686,10 +687,10 @@ static void jd_trace_atom_submit(struct kbase_context *const kctx, { struct kbase_device *const kbdev = kctx->kbdev; - KBASE_TLSTREAM_TL_NEW_ATOM(kbdev, katom, kbase_jd_atom_id(kctx, katom)); + KBASE_TLSTREAM_TL_NEW_ATOM(kbdev, katom, (u32)kbase_jd_atom_id(kctx, katom)); KBASE_TLSTREAM_TL_RET_ATOM_CTX(kbdev, katom, kctx); if (priority) - KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(kbdev, katom, *priority); + KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(kbdev, katom, (u32)*priority); KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_IDLE); kbase_kinstr_jm_atom_queue(katom); } @@ -1258,7 +1259,8 @@ void kbase_jd_done_worker(struct work_struct *data) return; } - if ((katom->event_code != BASE_JD_EVENT_DONE) && (!kbase_ctx_flag(katom->kctx, KCTX_DYING))) + if ((katom->event_code != BASE_JD_EVENT_DONE) && !kbase_ctx_flag(katom->kctx, KCTX_DYING) && + !kbase_ctx_flag(katom->kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) if (!kbase_is_quick_reset_enabled(kbdev)) dev_err(kbdev->dev, "t6xx: GPU fault 0x%02lx from job slot %d\n", (unsigned long)katom->event_code, katom->slot_nr); @@ -1445,7 +1447,7 @@ static void jd_cancel_worker(struct work_struct *data) * This can be called safely from atomic context. * The caller must hold kbdev->hwaccess_lock */ -void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp, +void kbase_jd_done(struct kbase_jd_atom *katom, unsigned int slot_nr, ktime_t *end_timestamp, kbasep_js_atom_done_code done_code) { struct kbase_context *kctx; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_jm.c index 15b0706e82d0..cac12df225a0 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_jm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_jm.c @@ -70,13 +70,13 @@ u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask) dev_dbg(kbdev->dev, "JM kick slot mask 0x%x\n", js_mask); while (js_mask) { - unsigned int js = ffs(js_mask) - 1; + unsigned int js = (unsigned int)ffs((int)js_mask) - 1U; int nr_jobs_to_submit = kbase_backend_slot_free(kbdev, js); if (kbase_jm_next_job(kbdev, js, nr_jobs_to_submit)) - ret_mask |= (1 << js); + ret_mask |= (1U << js); - js_mask &= ~(1 << js); + js_mask &= ~(1U << js); } dev_dbg(kbdev->dev, "Can still submit to mask 0x%x\n", ret_mask); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_jm.h index 977bcc8dcb92..fb0e69a0aee3 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_jm.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_jm.h @@ -49,7 +49,7 @@ u32 kbase_jm_kick(struct kbase_device *kbdev, u32 js_mask); */ static inline u32 kbase_jm_kick_all(struct kbase_device *kbdev) { - return kbase_jm_kick(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1); + return kbase_jm_kick(kbdev, (1U << kbdev->gpu_props.num_job_slots) - 1U); } /** diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js.c b/drivers/gpu/arm/bifrost/mali_kbase_js.c index 1dca014c82d4..55c1f4be25d5 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_js.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_js.c @@ -844,7 +844,7 @@ static bool kbase_jsctx_slot_prio_is_blocked(struct kbase_context *kctx, unsigne /* all bits of sched_prio or higher, with sched_prio = 0 being the * highest priority */ - higher_prios_mask = (prio_bit << 1) - 1u; + higher_prios_mask = (prio_bit << 1u) - 1u; return (slot_tracking->blocked & higher_prios_mask) != 0u; } @@ -972,7 +972,7 @@ static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, atomic_inc(&kbdev->js_data.nr_contexts_runnable); } } - kctx->slots_pullable |= (1 << js); + kctx->slots_pullable |= (1UL << js); return ret; } @@ -1013,7 +1013,7 @@ static bool kbase_js_ctx_list_add_pullable_head_nolock(struct kbase_device *kbde atomic_inc(&kbdev->js_data.nr_contexts_runnable); } } - kctx->slots_pullable |= (1 << js); + kctx->slots_pullable |= (1UL << js); return ret; } @@ -1083,7 +1083,7 @@ static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, atomic_dec(&kbdev->js_data.nr_contexts_runnable); } } - kctx->slots_pullable &= ~(1 << js); + kctx->slots_pullable &= ~(1UL << js); return ret; } @@ -1124,7 +1124,7 @@ static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, struct k atomic_dec(&kbdev->js_data.nr_contexts_runnable); } } - kctx->slots_pullable &= ~(1 << js); + kctx->slots_pullable &= ~(1UL << js); return ret; } @@ -1409,7 +1409,7 @@ void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority) /* Move kctx to the pullable/upullable list as per the new priority */ if (new_priority != kctx->priority) { for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { - if (kctx->slots_pullable & (1 << js)) + if (kctx->slots_pullable & (1UL << js)) list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], &kbdev->js_data.ctx_list_pullable[js][new_priority]); else @@ -1645,7 +1645,7 @@ bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom) * kick the job manager to attempt to fast-start the atom */ if (enqueue_required && kctx == kbdev->hwaccess.active_kctx[atom->slot_nr]) - kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); + kbase_jm_try_kick(kbdev, 1UL << atom->slot_nr); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); if (timer_sync) @@ -1885,7 +1885,7 @@ kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_ #endif kbase_pm_is_suspending(kbdev))) { int num_slots = kbdev->gpu_props.num_job_slots; - int slot; + unsigned int slot; /* Last reference, and we've been told to remove this context * from the Run Pool @@ -2402,7 +2402,8 @@ void kbasep_js_suspend(struct kbase_device *kbdev) void kbasep_js_resume(struct kbase_device *kbdev) { struct kbasep_js_device_data *js_devdata; - int js, prio; + unsigned int js; + int prio; KBASE_DEBUG_ASSERT(kbdev); js_devdata = &kbdev->js_data; @@ -2722,7 +2723,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js) } KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JS_PULL_JOB, kctx, katom, katom->jc, js, - katom->sched_priority); + (u64)katom->sched_priority); kbase_ctx_flag_set(kctx, KCTX_PULLED); kbase_ctx_flag_set(kctx, (KCTX_PULLED_SINCE_ACTIVE_JS0 << js)); @@ -2936,7 +2937,7 @@ static void js_return_worker(struct work_struct *data) struct kbasep_js_device_data *js_devdata = &kbdev->js_data; struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info; struct kbasep_js_atom_retained_state retained_state; - int js = katom->slot_nr; + unsigned int js = katom->slot_nr; bool slot_became_unblocked; bool timer_sync = false; bool context_idle = false; @@ -2991,8 +2992,8 @@ static void js_return_worker(struct work_struct *data) } if (kctx->as_nr != KBASEP_AS_NR_INVALID && !kbase_ctx_flag(kctx, KCTX_DYING)) { - int num_slots = kbdev->gpu_props.num_job_slots; - int slot; + unsigned int num_slots = kbdev->gpu_props.num_job_slots; + unsigned int slot; if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) kbasep_js_set_submit_allowed(js_devdata, kctx); @@ -3215,14 +3216,14 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom struct kbase_device *kbdev; unsigned long flags; bool timer_sync = false; - int atom_slot; + unsigned int atom_slot; bool context_idle = false; int prio = katom->sched_priority; kbdev = kctx->kbdev; atom_slot = katom->slot_nr; - dev_dbg(kbdev->dev, "%s for atom %pK (s:%d)\n", __func__, (void *)katom, atom_slot); + dev_dbg(kbdev->dev, "%s for atom %pK (s:%u)\n", __func__, (void *)katom, atom_slot); /* Update the incremental rendering state machine. */ @@ -3260,7 +3261,7 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom */ if (slot_became_unblocked) { dev_dbg(kbdev->dev, - "kctx %pK is no longer blocked from submitting on slot %d at priority %d or higher\n", + "kctx %pK is no longer blocked from submitting on slot %u at priority %d or higher\n", (void *)kctx, atom_slot, prio); if (kbase_js_ctx_pullable(kctx, atom_slot, true)) @@ -3529,7 +3530,7 @@ void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask) } while (js_mask) { - js = ffs(js_mask) - 1; + js = (unsigned int)ffs((int)js_mask) - 1; while (1) { struct kbase_context *kctx; @@ -3539,7 +3540,7 @@ void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask) kctx = kbase_js_ctx_list_pop_head(kbdev, js); if (!kctx) { - js_mask &= ~(1 << js); + js_mask &= ~(1UL << js); dev_dbg(kbdev->dev, "No kctx on pullable list (s:%u)\n", js); break; } @@ -3591,7 +3592,7 @@ void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask) } /* No more jobs can be submitted on this slot */ - js_mask &= ~(1 << js); + js_mask &= ~(1UL << js); break; } mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex); @@ -3599,9 +3600,9 @@ void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask) kbase_ctx_flag_clear(kctx, KCTX_PULLED); - if (!kbase_jm_kick(kbdev, 1 << js)) { + if (!kbase_jm_kick(kbdev, 1UL << js)) { dev_dbg(kbdev->dev, "No more jobs can be submitted (s:%u)\n", js); - js_mask &= ~(1 << js); + js_mask &= ~(1UL << js); } if (!kbase_ctx_flag(kctx, KCTX_PULLED)) { bool pullable; @@ -3649,7 +3650,7 @@ void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask) } mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex); - js_mask &= ~(1 << js); + js_mask &= ~(1UL << js); break; /* Could not run atoms on this slot */ } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c index c6a66be98178..aae4df83e98d 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c @@ -247,7 +247,7 @@ static int reader_changes_init(struct reader_changes *const changes, const size_ changes->threshold = min(((size_t)(changes->size)) / 4, ((size_t)(PAGE_SIZE)) / sizeof(*changes->data)); - return changes->size; + return (int)changes->size; } /** @@ -526,7 +526,7 @@ static ssize_t reader_changes_copy_to_user(struct reader_changes *const changes, buffer += read_size; buffer_size -= read_size; - ret += read_size; + ret += (ssize_t)read_size; changes_tail = (changes_tail + read_size / entry_size) & (changes->size - 1); smp_store_release(&changes->tail, changes_tail); } while (read_size); @@ -831,14 +831,14 @@ void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const katom) { struct kbase_context *const kctx = katom->kctx; struct kbase_device *const kbdev = kctx->kbdev; - const int slot = katom->slot_nr; + const unsigned int slot = katom->slot_nr; struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); BUILD_BUG_ON(SLOT_RB_SIZE != 2); lockdep_assert_held(&kbdev->hwaccess_lock); - if (WARN_ON(slot < 0 || slot >= GPU_MAX_JOB_SLOTS)) + if (WARN_ON(slot >= GPU_MAX_JOB_SLOTS)) return; if (WARN_ON(!submitted)) return; @@ -851,7 +851,7 @@ void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const katom) { struct kbase_context *const kctx = katom->kctx; struct kbase_device *const kbdev = kctx->kbdev; - const int slot = katom->slot_nr; + const unsigned int slot = katom->slot_nr; struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); struct kbase_jd_atom *const queued = kbase_gpu_inspect(kbdev, slot, 1); @@ -859,7 +859,7 @@ void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const katom) lockdep_assert_held(&kbdev->hwaccess_lock); - if (WARN_ON(slot < 0 || slot >= GPU_MAX_JOB_SLOTS)) + if (WARN_ON(slot >= GPU_MAX_JOB_SLOTS)) return; if (WARN_ON(!submitted)) return; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_prfcnt.c b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_prfcnt.c index a934948f11ee..ee7ea1487cf4 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_prfcnt.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_prfcnt.c @@ -114,7 +114,7 @@ struct kbase_kinstr_prfcnt_sample_array { * @scope: Scope of performance counters to capture. * @buffer_count: Number of buffers used to store samples. * @period_ns: Sampling period, in nanoseconds, or 0 if manual mode. - * @phys_em: Enable map used by the GPU. + * @enable_cm: Requested counter selection bitmasks. */ struct kbase_kinstr_prfcnt_client_config { u8 prfcnt_mode; @@ -122,7 +122,7 @@ struct kbase_kinstr_prfcnt_client_config { u8 scope; u16 buffer_count; u64 period_ns; - struct kbase_hwcnt_physical_enable_map phys_em; + struct kbase_hwcnt_enable_cm enable_cm; }; /** @@ -502,8 +502,7 @@ int kbasep_kinstr_prfcnt_set_block_meta_items(struct kbase_hwcnt_enable_map *ena return -EINVAL; metadata = dst->metadata; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { u8 *dst_blk; blk_stt_t hw_blk_stt; @@ -556,7 +555,7 @@ static void kbasep_kinstr_prfcnt_set_sample_metadata(struct kbase_kinstr_prfcnt_ /* PRFCNT_SAMPLE_META_TYPE_SAMPLE must be the first item */ ptr_md->hdr.item_type = PRFCNT_SAMPLE_META_TYPE_SAMPLE; ptr_md->hdr.item_version = PRFCNT_READER_API_VERSION; - ptr_md->u.sample_md.seq = atomic_read(&cli->write_idx); + ptr_md->u.sample_md.seq = (u64)atomic_read(&cli->write_idx); ptr_md->u.sample_md.flags = cli->sample_flags; /* Place the PRFCNT_SAMPLE_META_TYPE_CLOCK optionally as the 2nd */ @@ -592,8 +591,8 @@ static void kbasep_kinstr_prfcnt_set_sample_metadata(struct kbase_kinstr_prfcnt_ * @ts_end_ns: Time stamp for the end point of the sample dump. */ static void kbasep_kinstr_prfcnt_client_output_sample(struct kbase_kinstr_prfcnt_client *cli, - unsigned int buf_idx, u64 user_data, - u64 ts_start_ns, u64 ts_end_ns) + int buf_idx, u64 user_data, u64 ts_start_ns, + u64 ts_end_ns) { struct kbase_hwcnt_dump_buffer *dump_buf; struct kbase_hwcnt_dump_buffer *tmp_buf = &cli->tmp_buf; @@ -640,8 +639,8 @@ static int kbasep_kinstr_prfcnt_client_dump(struct kbase_kinstr_prfcnt_client *c int ret; u64 ts_start_ns = 0; u64 ts_end_ns = 0; - unsigned int write_idx; - unsigned int read_idx; + int write_idx; + int read_idx; size_t available_samples_count; WARN_ON(!cli); @@ -658,7 +657,7 @@ static int kbasep_kinstr_prfcnt_client_dump(struct kbase_kinstr_prfcnt_client *c WARN_ON(available_samples_count < 1); /* Reserve one slot to store the implicit sample taken on CMD_STOP */ available_samples_count -= 1; - if (write_idx - read_idx == available_samples_count) { + if ((size_t)(write_idx - read_idx) == available_samples_count) { /* For periodic sampling, the current active dump * will be accumulated in the next sample, when * a buffer becomes available. @@ -697,8 +696,8 @@ static int kbasep_kinstr_prfcnt_client_start(struct kbase_kinstr_prfcnt_client * { int ret; u64 tm_start, tm_end; - unsigned int write_idx; - unsigned int read_idx; + int write_idx; + int read_idx; size_t available_samples_count; WARN_ON(!cli); @@ -714,11 +713,11 @@ static int kbasep_kinstr_prfcnt_client_start(struct kbase_kinstr_prfcnt_client * /* Check whether there is space to store atleast an implicit sample * corresponding to CMD_STOP. */ - available_samples_count = cli->sample_count - (write_idx - read_idx); + available_samples_count = cli->sample_count - (size_t)(write_idx - read_idx); if (!available_samples_count) return -EBUSY; - kbase_hwcnt_gpu_enable_map_from_physical(&cli->enable_map, &cli->config.phys_em); + kbase_hwcnt_gpu_enable_map_from_cm(&cli->enable_map, &cli->config.enable_cm); /* Enable all the available clk_enable_map. */ cli->enable_map.clk_enable_map = (1ull << cli->kinstr_ctx->metadata->clk_cnt) - 1; @@ -747,10 +746,9 @@ static int kbasep_kinstr_prfcnt_client_stop(struct kbase_kinstr_prfcnt_client *c int ret; u64 tm_start = 0; u64 tm_end = 0; - struct kbase_hwcnt_physical_enable_map phys_em = { 0 }; size_t available_samples_count; - unsigned int write_idx; - unsigned int read_idx; + int write_idx; + int read_idx; WARN_ON(!cli); lockdep_assert_held(&cli->cmd_sync_lock); @@ -761,13 +759,14 @@ static int kbasep_kinstr_prfcnt_client_stop(struct kbase_kinstr_prfcnt_client *c mutex_lock(&cli->kinstr_ctx->lock); - kbase_hwcnt_gpu_enable_map_from_physical(&cli->enable_map, &phys_em); + /* Set enable map to 0 */ + kbase_hwcnt_gpu_enable_map_from_cm(&cli->enable_map, &(struct kbase_hwcnt_enable_cm){}); /* Check whether one has the buffer to hold the last sample */ write_idx = atomic_read(&cli->write_idx); read_idx = atomic_read(&cli->read_idx); - available_samples_count = cli->sample_count - (write_idx - read_idx); + available_samples_count = cli->sample_count - (size_t)(write_idx - read_idx); ret = kbase_hwcnt_virtualizer_client_set_counters(cli->hvcli, &cli->enable_map, &tm_start, &tm_end, &cli->tmp_buf); @@ -779,7 +778,7 @@ static int kbasep_kinstr_prfcnt_client_stop(struct kbase_kinstr_prfcnt_client *c if (!WARN_ON(!available_samples_count)) { write_idx %= cli->sample_arr.sample_count; /* Handle the last stop sample */ - kbase_hwcnt_gpu_enable_map_from_physical(&cli->enable_map, &cli->config.phys_em); + kbase_hwcnt_gpu_enable_map_from_cm(&cli->enable_map, &cli->config.enable_cm); /* As this is a stop sample, mark it as MANUAL */ kbasep_kinstr_prfcnt_client_output_sample(cli, write_idx, user_data, tm_start, tm_end); @@ -822,7 +821,7 @@ static int kbasep_kinstr_prfcnt_client_sync_dump(struct kbase_kinstr_prfcnt_clie static int kbasep_kinstr_prfcnt_client_discard(struct kbase_kinstr_prfcnt_client *cli) { - unsigned int write_idx; + int write_idx; WARN_ON(!cli); lockdep_assert_held(&cli->cmd_sync_lock); @@ -882,9 +881,9 @@ int kbasep_kinstr_prfcnt_cmd(struct kbase_kinstr_prfcnt_client *cli, static int kbasep_kinstr_prfcnt_get_sample(struct kbase_kinstr_prfcnt_client *cli, struct prfcnt_sample_access *sample_access) { - unsigned int write_idx; - unsigned int read_idx; - unsigned int fetch_idx; + int write_idx; + int read_idx; + int fetch_idx; u64 sample_offset_bytes; struct prfcnt_metadata *sample_meta; int err = 0; @@ -917,7 +916,7 @@ static int kbasep_kinstr_prfcnt_get_sample(struct kbase_kinstr_prfcnt_client *cl read_idx %= cli->sample_arr.sample_count; sample_meta = cli->sample_arr.samples[read_idx].sample_meta; - sample_offset_bytes = (u8 *)sample_meta - cli->sample_arr.user_buf; + sample_offset_bytes = (u64)((u8 *)sample_meta - cli->sample_arr.user_buf); sample_access->sequence = sample_meta->u.sample_md.seq; sample_access->sample_offset_bytes = sample_offset_bytes; @@ -933,9 +932,9 @@ error_out: static int kbasep_kinstr_prfcnt_put_sample(struct kbase_kinstr_prfcnt_client *cli, struct prfcnt_sample_access *sample_access) { - unsigned int write_idx; - unsigned int read_idx; - unsigned int fetch_idx; + int write_idx; + int read_idx; + int fetch_idx; u64 sample_offset_bytes; int err = 0; @@ -949,8 +948,8 @@ static int kbasep_kinstr_prfcnt_put_sample(struct kbase_kinstr_prfcnt_client *cl } read_idx %= cli->sample_arr.sample_count; - sample_offset_bytes = - (u8 *)cli->sample_arr.samples[read_idx].sample_meta - cli->sample_arr.user_buf; + sample_offset_bytes = (u64)((u8 *)cli->sample_arr.samples[read_idx].sample_meta - + cli->sample_arr.user_buf); if (sample_access->sample_offset_bytes != sample_offset_bytes) { err = -EINVAL; @@ -1148,8 +1147,7 @@ size_t kbasep_kinstr_prfcnt_get_sample_md_count(const struct kbase_hwcnt_metadat if (!metadata) return 0; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { /* Skip unavailable, non-enabled or reserved blocks */ if (kbase_kinstr_is_block_type_reserved(metadata, blk) || !kbase_hwcnt_metadata_block_instance_avail(metadata, blk, blk_inst) || @@ -1459,8 +1457,7 @@ static bool prfcnt_block_supported(const struct kbase_hwcnt_metadata *metadata, { size_t blk, blk_inst; - kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) - { + kbase_hwcnt_metadata_for_each_block(metadata, blk, blk_inst) { const enum kbase_hwcnt_gpu_v5_block_type blk_type = kbase_hwcnt_metadata_block_type(metadata, blk); const enum prfcnt_block_type prfcnt_block_type = @@ -1472,15 +1469,15 @@ static bool prfcnt_block_supported(const struct kbase_hwcnt_metadata *metadata, return false; } -static void kbasep_kinstr_prfcnt_block_enable_to_physical(uint32_t *phys_em, - const uint64_t *enable_mask) +static void kbasep_kinstr_prfcnt_block_enable_req_to_cfg(uint64_t *cfg_cm, + const uint64_t *enable_mask) { - /* Adding a baseline phys_em value '1' on any type that has been requested. This - * ensures the phys_em block states will always be reflected in the client's + /* Adding a baseline value '0xF' to cfg_cm on any type that has been requested. This + * ensures the block states will always be reflected in the client's * sample outputs, even when the client provided an all zero value mask. */ - *phys_em |= - (1 | kbase_hwcnt_backend_gpu_block_map_to_physical(enable_mask[0], enable_mask[1])); + cfg_cm[0] |= (0xF | enable_mask[0]); + cfg_cm[1] |= enable_mask[1]; } /** @@ -1558,28 +1555,28 @@ kbasep_kinstr_prfcnt_parse_request_enable(const struct prfcnt_request_enable *re */ switch (req_enable->block_type) { case PRFCNT_BLOCK_TYPE_FE: - kbasep_kinstr_prfcnt_block_enable_to_physical(&config->phys_em.fe_bm, - req_enable->enable_mask); + kbasep_kinstr_prfcnt_block_enable_req_to_cfg(config->enable_cm.fe_bm, + req_enable->enable_mask); break; case PRFCNT_BLOCK_TYPE_TILER: - kbasep_kinstr_prfcnt_block_enable_to_physical(&config->phys_em.tiler_bm, - req_enable->enable_mask); + kbasep_kinstr_prfcnt_block_enable_req_to_cfg(config->enable_cm.tiler_bm, + req_enable->enable_mask); break; case PRFCNT_BLOCK_TYPE_MEMORY: - kbasep_kinstr_prfcnt_block_enable_to_physical(&config->phys_em.mmu_l2_bm, - req_enable->enable_mask); + kbasep_kinstr_prfcnt_block_enable_req_to_cfg(config->enable_cm.mmu_l2_bm, + req_enable->enable_mask); break; case PRFCNT_BLOCK_TYPE_SHADER_CORE: - kbasep_kinstr_prfcnt_block_enable_to_physical(&config->phys_em.shader_bm, - req_enable->enable_mask); + kbasep_kinstr_prfcnt_block_enable_req_to_cfg(config->enable_cm.shader_bm, + req_enable->enable_mask); break; case PRFCNT_BLOCK_TYPE_FW: - kbasep_kinstr_prfcnt_block_enable_to_physical(&config->phys_em.fw_bm, - req_enable->enable_mask); + kbasep_kinstr_prfcnt_block_enable_req_to_cfg(config->enable_cm.fw_bm, + req_enable->enable_mask); break; case PRFCNT_BLOCK_TYPE_CSG: - kbasep_kinstr_prfcnt_block_enable_to_physical(&config->phys_em.csg_bm, - req_enable->enable_mask); + kbasep_kinstr_prfcnt_block_enable_req_to_cfg(config->enable_cm.csg_bm, + req_enable->enable_mask); break; default: err = -EINVAL; @@ -1778,8 +1775,8 @@ int kbasep_kinstr_prfcnt_client_create(struct kbase_kinstr_prfcnt_context *kinst break; case KINSTR_PRFCNT_DUMP_BUFFER: - kbase_hwcnt_gpu_enable_map_from_physical(&cli->enable_map, - &cli->config.phys_em); + kbase_hwcnt_gpu_enable_map_from_cm(&cli->enable_map, + &cli->config.enable_cm); cli->sample_count = cli->config.buffer_count; cli->sample_size = @@ -1806,8 +1803,8 @@ int kbasep_kinstr_prfcnt_client_create(struct kbase_kinstr_prfcnt_context *kinst /* Set enable map to be 0 to prevent virtualizer to init and kick the * backend to count. */ - kbase_hwcnt_gpu_enable_map_from_physical( - &cli->enable_map, &(struct kbase_hwcnt_physical_enable_map){ 0 }); + kbase_hwcnt_gpu_enable_map_from_cm(&cli->enable_map, + &(struct kbase_hwcnt_enable_cm){}); err = kbase_hwcnt_virtualizer_client_create(kinstr_ctx->hvirt, &cli->enable_map, &cli->hvcli); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.c b/drivers/gpu/arm/bifrost/mali_kbase_mem.c index 329de89812c0..ddf6ea352e72 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.c @@ -122,7 +122,7 @@ static int get_large_page_conf(char *buffer, const struct kernel_param *kp) break; } - return sprintf(buffer, "%s\n", out); + return scnprintf(buffer, PAGE_SIZE, "%s\n", out); } static const struct kernel_param_ops large_page_config_params = { @@ -312,7 +312,7 @@ int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 size_t i = 0; unsigned long attr; unsigned long mask = ~KBASE_REG_MEMATTR_MASK; - unsigned long gwt_mask = ~0; + unsigned long gwt_mask = ~0UL; int group_id; struct kbase_mem_phy_alloc *alloc; @@ -671,10 +671,10 @@ void kbase_sync_single(struct kbase_context *kctx, struct tagged_addr t_cpu_pa, if (likely(cpu_pa == gpu_pa)) { dma_addr_t dma_addr; - BUG_ON(!cpu_page); - BUG_ON(offset + size > PAGE_SIZE); + WARN_ON(!cpu_page); + WARN_ON((size_t)offset + size > PAGE_SIZE); - dma_addr = kbase_dma_addr_from_tagged(t_cpu_pa) + offset; + dma_addr = kbase_dma_addr_from_tagged(t_cpu_pa) + (dma_addr_t)offset; if (sync_fn == KBASE_SYNC_TO_CPU) dma_sync_single_for_cpu(kctx->kbdev->dev, dma_addr, size, @@ -692,7 +692,7 @@ void kbase_sync_single(struct kbase_context *kctx, struct tagged_addr t_cpu_pa, return; gpu_page = pfn_to_page(PFN_DOWN(gpu_pa)); - dma_addr = kbase_dma_addr_from_tagged(t_gpu_pa) + offset; + dma_addr = kbase_dma_addr_from_tagged(t_gpu_pa) + (dma_addr_t)offset; if (sync_fn == KBASE_SYNC_TO_DEVICE) { src = ((unsigned char *)kbase_kmap(cpu_page)) + offset; @@ -788,7 +788,7 @@ static int kbase_do_syncset(struct kbase_context *kctx, struct basep_syncset *ss /* Sync first page */ sz = MIN(((size_t)PAGE_SIZE - offset), size); - kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off], offset, sz, sync_fn); + kbase_sync_single(kctx, cpu_pa[page_off], gpu_pa[page_off], (off_t)offset, sz, sync_fn); /* Calculate the size for last page */ sz = ((start + size - 1) & ~PAGE_MASK) + 1; @@ -945,7 +945,7 @@ int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) kbase_gpu_vm_lock(kctx); if (gpu_addr >= BASE_MEM_COOKIE_BASE && gpu_addr < BASE_MEM_FIRST_FREE_ADDRESS) { - int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); + unsigned int cookie = PFN_DOWN(gpu_addr - BASE_MEM_COOKIE_BASE); reg = kctx->pending_regions[cookie]; if (!reg) { @@ -1156,15 +1156,15 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa /* Check if we have enough pages requested so we can allocate a large * page (512 * 4KB = 2MB ) */ - if (kbdev->pagesize_2mb && nr_left >= NUM_4K_PAGES_IN_2MB_PAGE) { - int nr_lp = nr_left / NUM_4K_PAGES_IN_2MB_PAGE; + if (kbdev->pagesize_2mb && nr_left >= NUM_PAGES_IN_2MB_LARGE_PAGE) { + size_t nr_lp = nr_left / NUM_PAGES_IN_2MB_LARGE_PAGE; res = kbase_mem_pool_alloc_pages(&kctx->mem_pools.large[alloc->group_id], - nr_lp * NUM_4K_PAGES_IN_2MB_PAGE, tp, true, + nr_lp * NUM_PAGES_IN_2MB_LARGE_PAGE, tp, true, kctx->task); if (res > 0) { - nr_left -= res; + nr_left -= (size_t)res; tp += res; nr_pages_to_account += res; } @@ -1175,17 +1175,18 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa spin_lock(&kctx->mem_partials_lock); list_for_each_entry_safe(sa, temp_sa, &kctx->mem_partials, link) { - int pidx = 0; + unsigned int pidx = 0; while (nr_left) { - pidx = find_next_zero_bit(sa->sub_pages, - NUM_4K_PAGES_IN_2MB_PAGE, pidx); + pidx = find_next_zero_bit( + sa->sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE, pidx); bitmap_set(sa->sub_pages, pidx, 1); *tp++ = as_tagged_tag(page_to_phys(sa->page + pidx), FROM_PARTIAL); nr_left--; - if (bitmap_full(sa->sub_pages, NUM_4K_PAGES_IN_2MB_PAGE)) { + if (bitmap_full(sa->sub_pages, + NUM_PAGES_IN_2MB_LARGE_PAGE)) { /* unlink from partial list when full */ list_del_init(&sa->link); break; @@ -1198,7 +1199,7 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa /* only if we actually have a chunk left <512. If more it indicates * that we couldn't allocate a 2MB above, so no point to retry here. */ - if (nr_left > 0 && nr_left < NUM_4K_PAGES_IN_2MB_PAGE) { + if (nr_left > 0 && nr_left < NUM_PAGES_IN_2MB_LARGE_PAGE) { /* create a new partial and suballocate the rest from it */ struct page *np = NULL; @@ -1229,10 +1230,10 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa /* store pointers back to the control struct */ np->lru.next = (void *)sa; - for (p = np; p < np + NUM_4K_PAGES_IN_2MB_PAGE; p++) + for (p = np; p < np + NUM_PAGES_IN_2MB_LARGE_PAGE; p++) p->lru.prev = (void *)np; INIT_LIST_HEAD(&sa->link); - bitmap_zero(sa->sub_pages, NUM_4K_PAGES_IN_2MB_PAGE); + bitmap_zero(sa->sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE); sa->page = np; for (i = 0; i < nr_left; i++) @@ -1245,7 +1246,7 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa * for the whole of the large page, and not just for the * sub-pages that have been used. */ - nr_pages_to_account += NUM_4K_PAGES_IN_2MB_PAGE; + nr_pages_to_account += NUM_PAGES_IN_2MB_LARGE_PAGE; /* expose for later use */ spin_lock(&kctx->mem_partials_lock); @@ -1281,7 +1282,7 @@ done: alloc_failed: /* The first step of error recovery is freeing any allocation that * might have succeeded. The function can be in this condition only - * in one case: it tried to allocate a combination of 2 MB and 4 kB + * in one case: it tried to allocate a combination of 2 MB and small * pages but only the former step succeeded. In this case, calculate * the number of 2 MB pages to release and free them. */ @@ -1293,8 +1294,8 @@ alloc_failed: } /* Undo the preliminary memory accounting that was done early on - * in the function. If only 4 kB pages are used: nr_left is equal - * to nr_pages_requested. If a combination of 2 MB and 4 kB was + * in the function. If only small pages are used: nr_left is equal + * to nr_pages_requested. If a combination of 2 MB and small pages was * attempted: nr_pages_requested is equal to the sum of nr_left * and nr_pages_to_free, and the latter has already been freed above. * @@ -1321,13 +1322,13 @@ static size_t free_partial_locked(struct kbase_context *kctx, struct kbase_mem_p head_page = (struct page *)p->lru.prev; sa = (struct kbase_sub_alloc *)head_page->lru.next; clear_bit(p - head_page, sa->sub_pages); - if (bitmap_empty(sa->sub_pages, NUM_4K_PAGES_IN_2MB_PAGE)) { + if (bitmap_empty(sa->sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE)) { list_del(&sa->link); kbase_mem_pool_free_locked(pool, head_page, true); kfree(sa); - nr_pages_to_account = NUM_4K_PAGES_IN_2MB_PAGE; - } else if (bitmap_weight(sa->sub_pages, NUM_4K_PAGES_IN_2MB_PAGE) == - NUM_4K_PAGES_IN_2MB_PAGE - 1) { + nr_pages_to_account = NUM_PAGES_IN_2MB_LARGE_PAGE; + } else if (bitmap_weight(sa->sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE) == + NUM_PAGES_IN_2MB_LARGE_PAGE - 1) { /* expose the partial again */ list_add(&sa->link, &kctx->mem_partials); } @@ -1386,12 +1387,13 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked(struct kbase_mem_phy_all new_pages = tp; if (kbdev->pagesize_2mb && pool->order) { - int nr_lp = nr_left / NUM_4K_PAGES_IN_2MB_PAGE; + size_t nr_lp = nr_left / NUM_PAGES_IN_2MB_LARGE_PAGE; - res = kbase_mem_pool_alloc_pages_locked(pool, nr_lp * NUM_4K_PAGES_IN_2MB_PAGE, tp); + res = kbase_mem_pool_alloc_pages_locked(pool, nr_lp * NUM_PAGES_IN_2MB_LARGE_PAGE, + tp); if (res > 0) { - nr_left -= res; + nr_left -= (size_t)res; tp += res; nr_pages_to_account += res; } @@ -1400,17 +1402,18 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked(struct kbase_mem_phy_all struct kbase_sub_alloc *sa, *temp_sa; list_for_each_entry_safe(sa, temp_sa, &kctx->mem_partials, link) { - int pidx = 0; + unsigned int pidx = 0; while (nr_left) { - pidx = find_next_zero_bit(sa->sub_pages, - NUM_4K_PAGES_IN_2MB_PAGE, pidx); + pidx = find_next_zero_bit( + sa->sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE, pidx); bitmap_set(sa->sub_pages, pidx, 1); *tp++ = as_tagged_tag(page_to_phys(sa->page + pidx), FROM_PARTIAL); nr_left--; - if (bitmap_full(sa->sub_pages, NUM_4K_PAGES_IN_2MB_PAGE)) { + if (bitmap_full(sa->sub_pages, + NUM_PAGES_IN_2MB_LARGE_PAGE)) { /* unlink from partial list when * full */ @@ -1425,7 +1428,7 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked(struct kbase_mem_phy_all * indicates that we couldn't allocate a 2MB above, so no point * to retry here. */ - if (nr_left > 0 && nr_left < NUM_4K_PAGES_IN_2MB_PAGE) { + if (nr_left > 0 && nr_left < NUM_PAGES_IN_2MB_LARGE_PAGE) { /* create a new partial and suballocate the rest from it */ struct page *np = NULL; @@ -1439,10 +1442,10 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked(struct kbase_mem_phy_all /* store pointers back to the control struct */ np->lru.next = (void *)sa; - for (p = np; p < np + NUM_4K_PAGES_IN_2MB_PAGE; p++) + for (p = np; p < np + NUM_PAGES_IN_2MB_LARGE_PAGE; p++) p->lru.prev = (void *)np; INIT_LIST_HEAD(&sa->link); - bitmap_zero(sa->sub_pages, NUM_4K_PAGES_IN_2MB_PAGE); + bitmap_zero(sa->sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE); sa->page = np; for (i = 0; i < nr_left; i++) @@ -1455,7 +1458,7 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked(struct kbase_mem_phy_all * for the whole of the large page, and not just for the * sub-pages that have been used. */ - nr_pages_to_account += NUM_4K_PAGES_IN_2MB_PAGE; + nr_pages_to_account += NUM_PAGES_IN_2MB_LARGE_PAGE; /* Indicate to user that we'll free this memory * later. @@ -1491,7 +1494,7 @@ done: alloc_failed: /* The first step of error recovery is freeing any allocation that * might have succeeded. The function can be in this condition only - * in one case: it tried to allocate a combination of 2 MB and 4 kB + * in one case: it tried to allocate a combination of 2 MB and small * pages but only the former step succeeded. In this case, calculate * the number of 2 MB pages to release and free them. */ @@ -1503,13 +1506,12 @@ alloc_failed: if (kbdev->pagesize_2mb && pool->order) { while (nr_pages_to_free) { if (is_huge_head(*start_free)) { - kbase_mem_pool_free_pages_locked(pool, - NUM_4K_PAGES_IN_2MB_PAGE, - start_free, - false, /* not dirty */ - true); /* return to pool */ - nr_pages_to_free -= NUM_4K_PAGES_IN_2MB_PAGE; - start_free += NUM_4K_PAGES_IN_2MB_PAGE; + kbase_mem_pool_free_pages_locked( + pool, NUM_PAGES_IN_2MB_LARGE_PAGE, start_free, + false, /* not dirty */ + true); /* return to pool */ + nr_pages_to_free -= NUM_PAGES_IN_2MB_LARGE_PAGE; + start_free += NUM_PAGES_IN_2MB_LARGE_PAGE; } else if (is_partial(*start_free)) { free_partial_locked(kctx, pool, *start_free); nr_pages_to_free--; @@ -1545,13 +1547,13 @@ static size_t free_partial(struct kbase_context *kctx, int group_id, struct tagg sa = (struct kbase_sub_alloc *)head_page->lru.next; spin_lock(&kctx->mem_partials_lock); clear_bit(p - head_page, sa->sub_pages); - if (bitmap_empty(sa->sub_pages, NUM_4K_PAGES_IN_2MB_PAGE)) { + if (bitmap_empty(sa->sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE)) { list_del(&sa->link); kbase_mem_pool_free(&kctx->mem_pools.large[group_id], head_page, true); kfree(sa); - nr_pages_to_account = NUM_4K_PAGES_IN_2MB_PAGE; - } else if (bitmap_weight(sa->sub_pages, NUM_4K_PAGES_IN_2MB_PAGE) == - NUM_4K_PAGES_IN_2MB_PAGE - 1) { + nr_pages_to_account = NUM_PAGES_IN_2MB_LARGE_PAGE; + } else if (bitmap_weight(sa->sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE) == + NUM_PAGES_IN_2MB_LARGE_PAGE - 1) { /* expose the partial again */ list_add(&sa->link, &kctx->mem_partials); } @@ -1603,12 +1605,12 @@ int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pag * it points to */ kbase_mem_pool_free_pages(&kctx->mem_pools.large[alloc->group_id], - NUM_4K_PAGES_IN_2MB_PAGE, start_free, syncback, + NUM_PAGES_IN_2MB_LARGE_PAGE, start_free, syncback, reclaimed); - nr_pages_to_free -= NUM_4K_PAGES_IN_2MB_PAGE; - start_free += NUM_4K_PAGES_IN_2MB_PAGE; - freed += NUM_4K_PAGES_IN_2MB_PAGE; - nr_pages_to_account += NUM_4K_PAGES_IN_2MB_PAGE; + nr_pages_to_free -= NUM_PAGES_IN_2MB_LARGE_PAGE; + start_free += NUM_PAGES_IN_2MB_LARGE_PAGE; + freed += NUM_PAGES_IN_2MB_LARGE_PAGE; + nr_pages_to_account += NUM_PAGES_IN_2MB_LARGE_PAGE; } else if (is_partial(*start_free)) { nr_pages_to_account += free_partial(kctx, alloc->group_id, *start_free); nr_pages_to_free--; @@ -1624,11 +1626,11 @@ int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pag nr_pages_to_free--; } kbase_mem_pool_free_pages(&kctx->mem_pools.small[alloc->group_id], - local_end_free - start_free, start_free, syncback, - reclaimed); - freed += local_end_free - start_free; + (size_t)(local_end_free - start_free), start_free, + syncback, reclaimed); + freed += (size_t)(local_end_free - start_free); + nr_pages_to_account += (size_t)(local_end_free - start_free); start_free += local_end_free - start_free; - nr_pages_to_account += freed; } } @@ -1708,12 +1710,12 @@ void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, * it points to */ WARN_ON(!pool->order); - kbase_mem_pool_free_pages_locked(pool, NUM_4K_PAGES_IN_2MB_PAGE, start_free, - syncback, false); - nr_pages_to_free -= NUM_4K_PAGES_IN_2MB_PAGE; - start_free += NUM_4K_PAGES_IN_2MB_PAGE; - freed += NUM_4K_PAGES_IN_2MB_PAGE; - nr_pages_to_account += NUM_4K_PAGES_IN_2MB_PAGE; + kbase_mem_pool_free_pages_locked(pool, NUM_PAGES_IN_2MB_LARGE_PAGE, + start_free, syncback, false); + nr_pages_to_free -= NUM_PAGES_IN_2MB_LARGE_PAGE; + start_free += NUM_PAGES_IN_2MB_LARGE_PAGE; + freed += NUM_PAGES_IN_2MB_LARGE_PAGE; + nr_pages_to_account += NUM_PAGES_IN_2MB_LARGE_PAGE; } else if (is_partial(*start_free)) { WARN_ON(!pool->order); nr_pages_to_account += free_partial_locked(kctx, pool, *start_free); @@ -1730,11 +1732,12 @@ void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, local_end_free++; nr_pages_to_free--; } - kbase_mem_pool_free_pages_locked(pool, local_end_free - start_free, + kbase_mem_pool_free_pages_locked(pool, + (size_t)(local_end_free - start_free), start_free, syncback, false); - freed += local_end_free - start_free; + freed += (size_t)(local_end_free - start_free); + nr_pages_to_account += (size_t)(local_end_free - start_free); start_free += local_end_free - start_free; - nr_pages_to_account += freed; } } @@ -1795,8 +1798,14 @@ void kbase_mem_kref_free(struct kref *kref) WARN_ONCE(alloc->imported.umm.current_mapping_usage_count != 1, "WARNING: expected exactly 1 mapping, got %d", alloc->imported.umm.current_mapping_usage_count); +#if (KERNEL_VERSION(6, 1, 55) <= LINUX_VERSION_CODE) + dma_buf_unmap_attachment_unlocked(alloc->imported.umm.dma_attachment, + alloc->imported.umm.sgt, + DMA_BIDIRECTIONAL); +#else dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); +#endif kbase_remove_dma_buf_usage(alloc->imported.umm.kctx, alloc); } dma_buf_detach(alloc->imported.umm.dma_buf, alloc->imported.umm.dma_attachment); @@ -1897,7 +1906,7 @@ void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc, struct tagged_addr phys = alloc->pages[i]; struct kbase_page_metadata *page_md = kbase_page_private(as_page(phys)); - /* Skip the 4KB page that is part of a large page, as the large page is + /* Skip the small page that is part of a large page, as the large page is * excluded from the migration process. */ if (is_huge(phys) || is_partial(phys)) @@ -2036,7 +2045,7 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, u64 u64 commit_pages, u64 large_extension) { struct device *dev = kctx->kbdev->dev; - int gpu_pc_bits = kctx->kbdev->gpu_props.log2_program_counter_size; + u32 gpu_pc_bits = kctx->kbdev->gpu_props.log2_program_counter_size; u64 gpu_pc_pages_max = 1ULL << gpu_pc_bits >> PAGE_SHIFT; struct kbase_va_region test_reg; @@ -2210,8 +2219,8 @@ static ssize_t kbase_jit_debugfs_common_read(struct file *file, char __user *buf goto out_unlock; } - size = scnprintf(data->buffer, sizeof(data->buffer), "%llu,%llu,%llu\n", - data->active_value, data->pool_value, data->destroy_value); + size = (size_t)scnprintf(data->buffer, sizeof(data->buffer), "%llu,%llu,%llu\n", + data->active_value, data->pool_value, data->destroy_value); } ret = simple_read_from_buffer(buf, len, ppos, data->buffer, size); @@ -2456,7 +2465,7 @@ static void kbase_jit_destroy_worker(struct work_struct *work) * by implementing "free on putting the last reference", * but only for JIT regions. */ - WARN_ON(atomic_read(®->no_user_free_count) > 1); + WARN_ON(atomic64_read(®->no_user_free_count) > 1); kbase_va_region_no_user_free_dec(reg); kbase_mem_free_region(kctx, reg); kbase_gpu_vm_unlock(kctx); @@ -2695,18 +2704,15 @@ static int kbase_jit_grow(struct kbase_context *kctx, const struct base_jit_allo if (reg->gpu_alloc->nents >= info->commit_pages) goto done; - /* Grow the backing */ - old_size = reg->gpu_alloc->nents; - /* Allocate some more pages */ delta = info->commit_pages - reg->gpu_alloc->nents; pages_required = delta; - if (kctx->kbdev->pagesize_2mb && pages_required >= NUM_4K_PAGES_IN_2MB_PAGE) { + if (kctx->kbdev->pagesize_2mb && pages_required >= NUM_PAGES_IN_2MB_LARGE_PAGE) { pool = &kctx->mem_pools.large[kctx->jit_group_id]; /* Round up to number of 2 MB pages required */ - pages_required += (NUM_4K_PAGES_IN_2MB_PAGE - 1); - pages_required /= NUM_4K_PAGES_IN_2MB_PAGE; + pages_required += (NUM_PAGES_IN_2MB_LARGE_PAGE - 1); + pages_required /= NUM_PAGES_IN_2MB_LARGE_PAGE; } else { pool = &kctx->mem_pools.small[kctx->jit_group_id]; } @@ -2723,7 +2729,7 @@ static int kbase_jit_grow(struct kbase_context *kctx, const struct base_jit_allo * between the grow and allocation. */ while (kbase_mem_pool_size(pool) < pages_required) { - int pool_delta = pages_required - kbase_mem_pool_size(pool); + size_t pool_delta = pages_required - kbase_mem_pool_size(pool); int ret; kbase_mem_pool_unlock(pool); @@ -2740,6 +2746,17 @@ static int kbase_jit_grow(struct kbase_context *kctx, const struct base_jit_allo kbase_mem_pool_lock(pool); } + if (reg->gpu_alloc->nents > info->commit_pages) { + kbase_mem_pool_unlock(pool); + spin_unlock(&kctx->mem_partials_lock); + dev_warn( + kctx->kbdev->dev, + "JIT alloc grown beyond the required number of initially required pages, this grow no longer needed."); + goto done; + } + + old_size = reg->gpu_alloc->nents; + delta = info->commit_pages - old_size; gpu_pages = kbase_alloc_phy_pages_helper_locked(reg->gpu_alloc, pool, delta, &prealloc_sas[0]); if (!gpu_pages) { @@ -3170,7 +3187,7 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, * flags. */ kbase_gpu_vm_lock(kctx); - if (unlikely(atomic_read(®->no_user_free_count) > 1)) { + if (unlikely(atomic64_read(®->no_user_free_count) > 1)) { kbase_gpu_vm_unlock(kctx); dev_err(kctx->kbdev->dev, "JIT region has no_user_free_count > 1!\n"); @@ -3228,7 +3245,7 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) * commit size */ u64 new_size = MAX(reg->initial_commit, - div_u64(old_pages * (100 - kctx->trim_level), 100)); + div_u64(old_pages * (100ULL - kctx->trim_level), 100ULL)); u64 delta = old_pages - new_size; if (delta) { @@ -3326,7 +3343,7 @@ bool kbase_jit_evict(struct kbase_context *kctx) * by implementing "free on putting the last reference", * but only for JIT regions. */ - WARN_ON(atomic_read(®->no_user_free_count) > 1); + WARN_ON(atomic64_read(®->no_user_free_count) > 1); kbase_va_region_no_user_free_dec(reg); kbase_mem_free_region(kctx, reg); } @@ -3354,7 +3371,7 @@ void kbase_jit_term(struct kbase_context *kctx) * by implementing "free on putting the last reference", * but only for JIT regions. */ - WARN_ON(atomic_read(&walker->no_user_free_count) > 1); + WARN_ON(atomic64_read(&walker->no_user_free_count) > 1); kbase_va_region_no_user_free_dec(walker); kbase_mem_free_region(kctx, walker); mutex_lock(&kctx->jit_evict_lock); @@ -3372,7 +3389,7 @@ void kbase_jit_term(struct kbase_context *kctx) * by implementing "free on putting the last reference", * but only for JIT regions. */ - WARN_ON(atomic_read(&walker->no_user_free_count) > 1); + WARN_ON(atomic64_read(&walker->no_user_free_count) > 1); kbase_va_region_no_user_free_dec(walker); kbase_mem_free_region(kctx, walker); mutex_lock(&kctx->jit_evict_lock); @@ -3518,7 +3535,7 @@ int kbase_user_buf_pin_pages(struct kbase_context *kctx, struct kbase_va_region */ for (i = 0; i < pinned_pages; i++) pa[i] = as_tagged(page_to_phys(pages[i])); - alloc->nents = pinned_pages; + alloc->nents = (size_t)pinned_pages; return 0; } @@ -3547,7 +3564,7 @@ int kbase_user_buf_dma_map_pages(struct kbase_context *kctx, struct kbase_va_reg struct page **pages = alloc->imported.user_buf.pages; struct device *dev = kctx->kbdev->dev; int write; - long i, pinned_pages, dma_mapped_pages; + size_t i, pinned_pages, dma_mapped_pages; enum dma_data_direction dma_dir; if (WARN_ON(alloc->type != KBASE_MEM_TYPE_IMPORTED_USER_BUF)) @@ -3624,12 +3641,12 @@ unwind_dma_map: */ static int kbase_user_buf_map(struct kbase_context *kctx, struct kbase_va_region *reg) { - long pinned_pages = 0; + size_t pinned_pages = 0; struct kbase_mem_phy_alloc *alloc; struct page **pages; struct tagged_addr *pa; - long i; - unsigned long gwt_mask = ~0; + size_t i; + unsigned long gwt_mask = ~0UL; int ret; /* Calls to this function are inherently asynchronous, with respect to * MMU operations. @@ -3922,7 +3939,7 @@ int kbase_map_external_resource(struct kbase_context *kctx, struct kbase_va_regi err = kbase_user_buf_from_dma_mapped_to_gpu_mapped(kctx, reg); if (err) - goto user_buf_to_gpu_mapped_fail; + return err; break; } @@ -3961,9 +3978,6 @@ int kbase_map_external_resource(struct kbase_context *kctx, struct kbase_va_regi kbase_va_region_alloc_get(kctx, reg); kbase_mem_phy_alloc_get(alloc); return 0; - -user_buf_to_gpu_mapped_fail: - return err; } void kbase_unmap_external_resource(struct kbase_context *kctx, struct kbase_va_region *reg) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.h b/drivers/gpu/arm/bifrost/mali_kbase_mem.h index a8e59b65a73d..e4a7d6bd0a30 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.h @@ -31,7 +31,6 @@ #endif #include -#include #include #include #include "mali_kbase_pm.h" @@ -40,7 +39,10 @@ #include "mali_kbase_mem_linux.h" #include "mali_kbase_reg_track.h" #include "mali_kbase_mem_migrate.h" -#include "mali_kbase_refcount_defs.h" + +#include +#include +#include static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages); @@ -107,49 +109,49 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int /* AS_MEMATTR values from MMU_MEMATTR_STAGE1: */ /* Use GPU implementation-defined caching policy. */ -#define KBASE_MEMATTR_IMPL_DEF_CACHE_POLICY \ - ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_IMPL) | \ - AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_WRITE_BACK))) +#define KBASE_MEMATTR_IMPL_DEF_CACHE_POLICY \ + ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_IMPL) | \ + AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_WRITE_BACK))) /* The attribute set to force all resources to be cached. */ -#define KBASE_MEMATTR_FORCE_TO_CACHE_ALL \ - ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_W_MASK | \ - AS_MEMATTR_ATTRIBUTE0_ALLOC_R_MASK | \ - AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_ALLOC) | \ - AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_WRITE_BACK))) +#define KBASE_MEMATTR_FORCE_TO_CACHE_ALL \ + ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_W_MASK | \ + AS_MEMATTR_ATTRIBUTE0_ALLOC_R_MASK | \ + AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_ALLOC) | \ + AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_WRITE_BACK))) /* Inner write-alloc cache setup, no outer caching */ -#define KBASE_MEMATTR_WRITE_ALLOC \ - ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_W_MASK | \ - AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_ALLOC) | \ - AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_WRITE_BACK))) +#define KBASE_MEMATTR_WRITE_ALLOC \ + ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_W_MASK | \ + AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_ALLOC) | \ + AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_WRITE_BACK))) /* Set to implementation defined, outer caching */ -#define KBASE_MEMATTR_AARCH64_OUTER_IMPL_DEF \ - ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_IMPL) | \ - AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_WRITE_BACK))) +#define KBASE_MEMATTR_AARCH64_OUTER_IMPL_DEF \ + ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_IMPL) | \ + AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_WRITE_BACK))) /* Set to write back memory, outer caching */ -#define KBASE_MEMATTR_AARCH64_OUTER_WA \ - ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_W_MASK | \ - AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_ALLOC) | \ - AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_WRITE_BACK))) +#define KBASE_MEMATTR_AARCH64_OUTER_WA \ + ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_W_MASK | \ + AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_ALLOC) | \ + AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_WRITE_BACK))) /* Set to inner non-cacheable, outer-non-cacheable * Setting defined by the alloc bits is ignored, but set to a valid encoding: * - no-alloc on read * - no alloc on write */ -#define KBASE_MEMATTR_AARCH64_NON_CACHEABLE \ - ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_ALLOC) | \ - AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_NON_CACHEABLE))) +#define KBASE_MEMATTR_AARCH64_NON_CACHEABLE \ + ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_ALLOC) | \ + AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_NON_CACHEABLE))) /* Symbols for default MEMATTR to use * Default is - HW implementation defined caching @@ -175,11 +177,11 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int * shared, otherwise inner non-cacheable. * Outer cacheable if inner or outer shared, otherwise outer non-cacheable. */ -#define KBASE_MEMATTR_AARCH64_SHARED \ - ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_IMPL) | \ - AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ - 0, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SHARED))) +#define KBASE_MEMATTR_AARCH64_SHARED \ + ((unsigned long long)(AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_ALLOC_SEL_IMPL) | \ + AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SET( \ + 0ull, AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SHARED))) /* Normal memory, shared between MCU and Host */ #define KBASE_MEMATTR_INDEX_SHARED 6 @@ -454,7 +456,7 @@ enum kbase_page_status { * @vmap_count: Counter of kernel mappings. * @group_id: Memory group ID obtained at the time of page allocation. * - * Each 4KB page will have a reference to this struct in the private field. + * Each small page will have a reference to this struct in the private field. * This will be used to keep track of information required for Linux page * migration functionality as well as address for DMA mapping. */ @@ -473,10 +475,12 @@ struct kbase_page_metadata { struct { struct kbase_va_region *reg; struct kbase_mmu_table *mmut; + /* GPU virtual page frame number, in GPU_PAGE_SIZE units */ u64 vpfn; } mapped; struct { struct kbase_mmu_table *mmut; + /* GPU virtual page frame number info is in GPU_PAGE_SIZE units */ u64 pgd_vpfn_level; } pt_mapped; struct { @@ -687,7 +691,7 @@ struct kbase_va_region { #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ kbase_refcount_t va_refcnt; - atomic_t no_user_free_count; + atomic64_t no_user_free_count; }; /* Special marker for failed JIT allocations that still must be marked as @@ -785,7 +789,7 @@ static inline struct kbase_va_region *kbase_va_region_alloc_put(struct kbase_con */ static inline bool kbase_va_region_is_no_user_free(struct kbase_va_region *region) { - return atomic_read(®ion->no_user_free_count) > 0; + return atomic64_read(®ion->no_user_free_count) > 0; } /** @@ -801,10 +805,10 @@ static inline bool kbase_va_region_is_no_user_free(struct kbase_va_region *regio static inline void kbase_va_region_no_user_free_inc(struct kbase_va_region *region) { WARN_ON(kbase_is_region_shrinkable(region)); - WARN_ON(atomic_read(®ion->no_user_free_count) == INT_MAX); + WARN_ON(atomic64_read(®ion->no_user_free_count) == S64_MAX); /* non-atomic as kctx->reg_lock is held */ - atomic_inc(®ion->no_user_free_count); + atomic64_inc(®ion->no_user_free_count); } /** @@ -816,7 +820,7 @@ static inline void kbase_va_region_no_user_free_dec(struct kbase_va_region *regi { WARN_ON(!kbase_va_region_is_no_user_free(region)); - atomic_dec(®ion->no_user_free_count); + atomic64_dec(®ion->no_user_free_count); } /* Common functions */ @@ -967,14 +971,14 @@ static inline int kbase_reg_prepare_native(struct kbase_va_region *reg, struct k #define KBASE_MEM_POOL_MAX_SIZE_KCTX (SZ_64M >> PAGE_SHIFT) /* - * The order required for a 2MB page allocation (2^order * 4KB = 2MB) + * The order required for a 2MB page allocation (2^order * PAGE_SIZE = 2MB) */ -#define KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER 9 +#define KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER (__builtin_ffs(NUM_PAGES_IN_2MB_LARGE_PAGE) - 1) /* - * The order required for a 4KB page allocation + * The order required for a small page allocation */ -#define KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER 0 +#define KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER 0 /** * kbase_mem_pool_config_set_max_size - Set maximum number of free pages in @@ -1009,7 +1013,7 @@ kbase_mem_pool_config_get_max_size(const struct kbase_mem_pool_config *const con * kbase_mem_pool_init - Create a memory pool for a kbase device * @pool: Memory pool to initialize * @config: Initial configuration for the memory pool - * @order: Page order for physical page size (order=0=>4kB, order=9=>2MB) + * @order: Page order for physical page size (order=0 => small page, order != 0 => 2MB) * @group_id: A memory group ID to be passed to a platform-specific * memory group manager, if present. * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). @@ -1108,7 +1112,7 @@ void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, boo /** * kbase_mem_pool_alloc_pages - Allocate pages from memory pool * @pool: Memory pool to allocate from - * @nr_4k_pages: Number of pages to allocate + * @nr_small_pages: Number of pages to allocate * @pages: Pointer to array where the physical address of the allocated * pages will be stored. * @partial_allowed: If fewer pages allocated is allowed @@ -1130,14 +1134,14 @@ void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, boo * the kernel OoM killer runs. If the caller must allocate pages while holding * this lock, it should use kbase_mem_pool_alloc_pages_locked() instead. */ -int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, +int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_small_pages, struct tagged_addr *pages, bool partial_allowed, struct task_struct *page_owner); /** * kbase_mem_pool_alloc_pages_locked - Allocate pages from memory pool * @pool: Memory pool to allocate from - * @nr_4k_pages: Number of pages to allocate + * @nr_small_pages: Number of pages to allocate * @pages: Pointer to array where the physical address of the allocated * pages will be stored. * @@ -1171,7 +1175,7 @@ int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, * * Note : Caller must hold the pool lock. */ -int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, size_t nr_4k_pages, +int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, size_t nr_small_pages, struct tagged_addr *pages); /** @@ -1249,7 +1253,9 @@ void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size); * Adds @nr_to_grow pages to the pool. Note that this may cause the pool to * become larger than the maximum size specified. * - * Return: 0 on success, -ENOMEM if unable to allocate sufficent pages + * Return: 0 on success, -ENOMEM if unable to allocate sufficient pages or + * -EPERM if the allocation of pages is not permitted due to the process exit + * or context termination. */ int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow, struct task_struct *page_owner); @@ -1500,7 +1506,7 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); * Return: The address of the buffer containing the MMU dump or NULL on error * (including if the @c nr_pages is too small) */ -void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); +void *kbase_mmu_dump(struct kbase_context *kctx, size_t nr_pages); #endif /** diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c index 522495ecaec0..9a30001634f1 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c @@ -84,6 +84,18 @@ #define IR_THRESHOLD_STEPS (256u) +/* + * fully_backed_gpf_memory - enable full physical backing of all grow-on-GPU-page-fault + * allocations in the kernel. + */ +static bool fully_backed_gpf_memory; +module_param(fully_backed_gpf_memory, bool, 0444); +MODULE_PARM_DESC( + fully_backed_gpf_memory, + "Enable the full physical backing of all grow-on-GPU-page-fault allocations in the kernel." + "Note that this should only be enabled for testing worst case memory usage " + "and should not be enabled in production"); + #if MALI_USE_CSF static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, struct vm_area_struct *vma); static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, struct vm_area_struct *vma); @@ -341,11 +353,13 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages if (kbase_check_alloc_sizes(kctx, *flags, va_pages, commit_pages, extension)) goto bad_sizes; -#ifdef CONFIG_MALI_MEMORY_FULLY_BACKED - /* Ensure that memory is fully physically-backed. */ - if (*flags & BASE_MEM_GROW_ON_GPF) - commit_pages = va_pages; -#endif + /* Ensure that memory is fully physically-backed if full physical backing + * has been requested for GPF allocations. + */ + if (unlikely(fully_backed_gpf_memory)) { + if (*flags & BASE_MEM_GROW_ON_GPF) + commit_pages = va_pages; + } /* find out which VA zone to use */ if (*flags & BASE_MEM_SAME_VA) @@ -391,7 +405,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages *flags &= ~BASE_MEM_CACHED_CPU; if (*flags & BASE_MEM_GROW_ON_GPF) { - unsigned int const ir_threshold = atomic_read(&kctx->kbdev->memdev.ir_threshold); + unsigned int const ir_threshold = + (unsigned int)atomic_read(&kctx->kbdev->memdev.ir_threshold); reg->threshold_pages = ((va_pages * ir_threshold) + (IR_THRESHOLD_STEPS / 2)) / IR_THRESHOLD_STEPS; @@ -463,8 +478,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages * the SAME_VA zone inside kbase_context_get_unmapped_area(). */ if (kctx->kbdev->gpu_props.mmu.va_bits > 33) { - if (va_pages >= (SZ_2M / SZ_4K)) - align = (SZ_2M / SZ_4K); + if (va_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE) + align = NUM_PAGES_IN_2MB_LARGE_PAGE; } if (*gpu_va) align = 1; @@ -664,7 +679,7 @@ static unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker * dev_err(kctx->kbdev->dev, "invalid evict_nents(%d)", evict_nents); nr_freeable_items = 0; } else { - nr_freeable_items = evict_nents; + nr_freeable_items = (unsigned long)evict_nents; } #if KERNEL_VERSION(4, 19, 0) <= LINUX_VERSION_CODE @@ -923,6 +938,10 @@ static int kbase_mem_flags_change_imported_umm(struct kbase_context *kctx, unsig if (flags & ~(BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL)) return ret; + /* Don't change outer sharability if platform is not coherent */ + if ((BASE_MEM_COHERENT_SYSTEM & flags) && !kbase_device_is_cpu_coherent(kctx->kbdev)) + return ret; + /* shareability flags are ignored for GPU uncached memory * instead of causing an error. */ @@ -1142,7 +1161,7 @@ int kbase_mem_do_sync_imported(struct kbase_context *kctx, struct kbase_va_regio reg->gpu_alloc->imported.umm.dma_attachment; struct sg_table *sgt = reg->gpu_alloc->imported.umm.sgt; - dma_sync_sg_for_device(attachment->dev, sgt->sgl, sgt->nents, dir); + dma_sync_sg_for_device(attachment->dev, sgt->sgl, (int)sgt->nents, dir); ret = 0; } #else @@ -1158,7 +1177,7 @@ int kbase_mem_do_sync_imported(struct kbase_context *kctx, struct kbase_va_regio reg->gpu_alloc->imported.umm.dma_attachment; struct sg_table *sgt = reg->gpu_alloc->imported.umm.sgt; - dma_sync_sg_for_cpu(attachment->dev, sgt->sgl, sgt->nents, dir); + dma_sync_sg_for_cpu(attachment->dev, sgt->sgl, (int)sgt->nents, dir); ret = 0; } #else @@ -1187,8 +1206,13 @@ static void kbase_mem_umm_unmap_attachment(struct kbase_context *kctx, { struct tagged_addr *pa = alloc->pages; +#if (KERNEL_VERSION(6, 1, 55) <= LINUX_VERSION_CODE) + dma_buf_unmap_attachment_unlocked(alloc->imported.umm.dma_attachment, + alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); +#else dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); +#endif alloc->imported.umm.sgt = NULL; kbase_remove_dma_buf_usage(kctx, alloc); @@ -1223,7 +1247,12 @@ static int kbase_mem_umm_map_attachment(struct kbase_context *kctx, struct kbase WARN_ON_ONCE(alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM); WARN_ON_ONCE(alloc->imported.umm.sgt); +#if (KERNEL_VERSION(6, 1, 55) <= LINUX_VERSION_CODE) + sgt = dma_buf_map_attachment_unlocked(alloc->imported.umm.dma_attachment, + DMA_BIDIRECTIONAL); +#else sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, DMA_BIDIRECTIONAL); +#endif if (IS_ERR_OR_NULL(sgt)) return -EINVAL; @@ -1272,7 +1301,7 @@ int kbase_mem_umm_map(struct kbase_context *kctx, struct kbase_va_region *reg) { int err; struct kbase_mem_phy_alloc *alloc; - unsigned long gwt_mask = ~0; + unsigned long gwt_mask = ~0UL; /* Calls to this function are inherently asynchronous, with respect to * MMU operations. @@ -1539,7 +1568,7 @@ no_alloc: u32 kbase_get_cache_line_alignment(struct kbase_device *kbdev) { - u32 cpu_cache_line_size = cache_line_size(); + u32 cpu_cache_line_size = (u32)cache_line_size(); u32 gpu_cache_line_size = (1UL << kbdev->gpu_props.log2_line_size); return ((cpu_cache_line_size > gpu_cache_line_size) ? cpu_cache_line_size : @@ -2068,8 +2097,8 @@ void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, struct kbase_va_re return; unmap_mapping_range(kctx->kfile->filp->f_inode->i_mapping, - (gpu_va_start + new_pages) << PAGE_SHIFT, - (old_pages - new_pages) << PAGE_SHIFT, 1); + (loff_t)(gpu_va_start + new_pages) << PAGE_SHIFT, + (loff_t)(old_pages - new_pages) << PAGE_SHIFT, 1); } int kbase_mem_shrink_gpu_mapping(struct kbase_context *const kctx, @@ -2157,14 +2186,22 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) if (kbase_va_region_is_no_user_free(reg)) goto out_unlock; -#ifdef CONFIG_MALI_MEMORY_FULLY_BACKED - /* Reject resizing commit size */ - if (reg->flags & KBASE_REG_PF_GROW) - new_pages = reg->nr_pages; -#endif + if (unlikely(fully_backed_gpf_memory)) { + if ((reg->flags & KBASE_REG_PF_GROW) && (reg->nr_pages != reg->initial_commit)) { + dev_err(kctx->kbdev->dev, + "GPF region has different physical and VA page size!"); + res = -EINVAL; + } else { + /* The condition should already be satisfied, so no need to change + * the commit size. + */ + res = 0; + } + goto out_unlock; + } + /* no change */ if (new_pages == reg->gpu_alloc->nents) { - /* no change */ res = 0; goto out_unlock; } @@ -2540,7 +2577,7 @@ static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct { struct kbase_va_region *new_reg; void *kaddr; - u32 nr_pages; + size_t nr_pages; size_t size; int err = 0; @@ -2860,10 +2897,10 @@ void kbase_sync_mem_regions(struct kbase_context *kctx, struct kbase_vmap_struct { size_t i; off_t const offset = map->offset_in_page; - size_t const page_count = PFN_UP(offset + map->size); + size_t const page_count = PFN_UP((size_t)offset + map->size); /* Sync first page */ - size_t sz = MIN(((size_t)PAGE_SIZE - offset), map->size); + size_t sz = MIN((PAGE_SIZE - (size_t)offset), map->size); struct tagged_addr cpu_pa = map->cpu_pages[0]; struct tagged_addr gpu_pa = map->gpu_pages[0]; @@ -2880,7 +2917,7 @@ void kbase_sync_mem_regions(struct kbase_context *kctx, struct kbase_vmap_struct if (page_count > 1) { cpu_pa = map->cpu_pages[page_count - 1]; gpu_pa = map->gpu_pages[page_count - 1]; - sz = ((offset + map->size - 1) & ~PAGE_MASK) + 1; + sz = (((size_t)offset + map->size - 1) & ~PAGE_MASK) + 1; kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, dest); } } @@ -2912,7 +2949,7 @@ static void kbase_vmap_phy_pages_migrate_count_increment(struct tagged_addr *pag struct page *p = as_page(pages[i]); struct kbase_page_metadata *page_md = kbase_page_private(p); - /* Skip the 4KB page that is part of a large page, as the large page is + /* Skip the small page that is part of a large page, as the large page is * excluded from the migration process. */ if (is_huge(pages[i]) || is_partial(pages[i])) @@ -2964,7 +3001,7 @@ static void kbase_vunmap_phy_pages_migrate_count_decrement(struct tagged_addr *p struct page *p = as_page(pages[i]); struct kbase_page_metadata *page_md = kbase_page_private(p); - /* Skip the 4KB page that is part of a large page, as the large page is + /* Skip the small page that is part of a large page, as the large page is * excluded from the migration process. */ if (is_huge(pages[i]) || is_partial(pages[i])) @@ -3021,7 +3058,7 @@ static int kbase_vmap_phy_pages(struct kbase_context *kctx, struct kbase_va_regi if ((vmap_flags & KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING) && (page_count > (KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES - - atomic_read(&kctx->permanent_mapped_pages)))) { + (size_t)atomic_read(&kctx->permanent_mapped_pages)))) { dev_warn( kctx->kbdev->dev, "Request for %llu more pages mem needing a permanent mapping would breach limit %lu, currently at %d pages", @@ -3172,7 +3209,7 @@ static void kbase_vunmap_phy_pages(struct kbase_context *kctx, struct kbase_vmap * haven't been released yet. */ if (kbase_is_page_migration_enabled() && !kbase_mem_is_imported(map->gpu_alloc->type)) { - const size_t page_count = PFN_UP(map->offset_in_page + map->size); + const size_t page_count = PFN_UP((size_t)map->offset_in_page + map->size); struct tagged_addr *pages_array = map->cpu_pages; kbase_vunmap_phy_pages_migrate_count_decrement(pages_array, page_count); @@ -3181,7 +3218,7 @@ static void kbase_vunmap_phy_pages(struct kbase_context *kctx, struct kbase_vmap if (map->flags & KBASE_VMAP_FLAG_SYNC_NEEDED) kbase_sync_mem_regions(kctx, map, KBASE_SYNC_TO_DEVICE); if (map->flags & KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING) { - size_t page_count = PFN_UP(map->offset_in_page + map->size); + size_t page_count = PFN_UP((size_t)map->offset_in_page + map->size); WARN_ON(page_count > (size_t)atomic_read(&kctx->permanent_mapped_pages)); atomic_sub(page_count, &kctx->permanent_mapped_pages); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h index cb3b5038554c..629ca3a2c07b 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_lowlevel.h @@ -28,6 +28,11 @@ #include +/* Kernel/CPU page size can be 4/16/64KB whereas GPU page size is fixed as 4KB */ +#define GPU_PAGE_SIZE SZ_4K +#define GPU_PAGE_MASK (~(SZ_4K - 1)) +#define GPU_PAGES_PER_CPU_PAGE (PAGE_SIZE / GPU_PAGE_SIZE) + /* Flags for kbase_phy_allocator_pages_alloc */ #define KBASE_PHY_PAGES_FLAG_DEFAULT (0) /** Default allocation flag */ #define KBASE_PHY_PAGES_FLAG_CLEAR (1 << 0) /** Clear the pages after allocation */ @@ -49,7 +54,7 @@ struct tagged_addr { #define HUGE_HEAD (1u << 1) #define FROM_PARTIAL (1u << 2) -#define NUM_4K_PAGES_IN_2MB_PAGE (SZ_2M / SZ_4K) +#define NUM_PAGES_IN_2MB_LARGE_PAGE (SZ_2M / PAGE_SIZE) #define KBASE_INVALID_PHYSICAL_ADDRESS (~(phys_addr_t)0 & PAGE_MASK) @@ -71,7 +76,7 @@ struct tagged_addr { */ static inline phys_addr_t as_phys_addr_t(struct tagged_addr t) { - return t.tagged_addr & PAGE_MASK; + return t.tagged_addr & GPU_PAGE_MASK; } /** @@ -90,7 +95,7 @@ static inline struct page *as_page(struct tagged_addr t) * there is no tag info present, the lower order 12 bits will be 0 * @phys: physical address to be converted to tagged type * - * This is used for 4KB physical pages allocated by the Driver or imported pages + * This is used for small physical pages allocated by the Driver or imported pages * and is needed as physical pages tracking object stores the reference for * physical pages using tagged address type in lieu of the type generally used * for physical addresses. @@ -101,7 +106,7 @@ static inline struct tagged_addr as_tagged(phys_addr_t phys) { struct tagged_addr t; - t.tagged_addr = phys & PAGE_MASK; + t.tagged_addr = phys & GPU_PAGE_MASK; return t; } @@ -119,12 +124,12 @@ static inline struct tagged_addr as_tagged_tag(phys_addr_t phys, int tag) { struct tagged_addr t; - t.tagged_addr = (phys & PAGE_MASK) | (tag & ~PAGE_MASK); + t.tagged_addr = (phys & GPU_PAGE_MASK) | (tag & ~GPU_PAGE_MASK); return t; } /** - * is_huge - Check if the physical page is one of the 512 4KB pages of the + * is_huge - Check if the physical page is one of the small pages of the * large page which was not split to be used partially * @t: tagged address storing the tag in the lower order bits. * @@ -136,9 +141,8 @@ static inline bool is_huge(struct tagged_addr t) } /** - * is_huge_head - Check if the physical page is the first 4KB page of the - * 512 4KB pages within a large page which was not split - * to be used partially + * is_huge_head - Check if the physical page is the first small page within + * a large page which was not split to be used partially * @t: tagged address storing the tag in the lower order bits. * * Return: true if page is the first page of a large page, or false @@ -151,8 +155,8 @@ static inline bool is_huge_head(struct tagged_addr t) } /** - * is_partial - Check if the physical page is one of the 512 pages of the - * large page which was split in 4KB pages to be used + * is_partial - Check if the physical page is one of the small pages of the + * large page which was split in small pages to be used * partially for allocations >= 2 MB in size. * @t: tagged address storing the tag in the lower order bits. * @@ -164,19 +168,19 @@ static inline bool is_partial(struct tagged_addr t) } /** - * index_in_large_page() - Get index of a 4KB page within a 2MB page which + * index_in_large_page() - Get index of a small page within a 2MB page which * wasn't split to be used partially. * - * @t: Tagged physical address of the physical 4KB page that lies within + * @t: Tagged physical address of the physical small page that lies within * the large (or 2 MB) physical page. * - * Return: Index of the 4KB page within a 2MB page + * Return: Index of the small page within a 2MB page */ static inline unsigned int index_in_large_page(struct tagged_addr t) { WARN_ON(!is_huge(t)); - return (PFN_DOWN(as_phys_addr_t(t)) & (NUM_4K_PAGES_IN_2MB_PAGE - 1)); + return (PFN_DOWN(as_phys_addr_t(t)) & (NUM_PAGES_IN_2MB_LARGE_PAGE - 1)); } /** diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c index c59036201fbe..93a07e7db4fa 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c @@ -297,7 +297,9 @@ static int kbasep_migrate_page_allocated_mapped(struct page *old_page, struct pa /* Unmap the old physical range. */ unmap_mapping_range(kctx->kfile->filp->f_inode->i_mapping, - page_md->data.mapped.vpfn << PAGE_SHIFT, PAGE_SIZE, 1); + (loff_t)(page_md->data.mapped.vpfn / GPU_PAGES_PER_CPU_PAGE) + << PAGE_SHIFT, + PAGE_SIZE, 1); ret = kbase_mmu_migrate_page(as_tagged(page_to_phys(old_page)), as_tagged(page_to_phys(new_page)), old_dma_addr, new_dma_addr, diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.h index a51a6ce19754..ece8734de792 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.h @@ -21,6 +21,14 @@ #ifndef _KBASE_MEM_MIGRATE_H #define _KBASE_MEM_MIGRATE_H +#include + +#include + +struct kbase_device; +struct file; +struct page; + /** * DOC: Base kernel page migration implementation. */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c index 159d84042366..cb862d5b029c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool.c @@ -79,8 +79,8 @@ static inline bool can_alloc_page(struct kbase_mem_pool *pool, struct task_struc static size_t kbase_mem_pool_capacity(struct kbase_mem_pool *pool) { - ssize_t max_size = kbase_mem_pool_max_size(pool); - ssize_t cur_size = kbase_mem_pool_size(pool); + ssize_t max_size = (ssize_t)kbase_mem_pool_max_size(pool); + ssize_t cur_size = (ssize_t)kbase_mem_pool_size(pool); return max(max_size - cur_size, (ssize_t)0); } @@ -299,7 +299,7 @@ struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool, const bool alloc_ return NULL; } - /* Setup page metadata for 4KB pages when page migration is enabled */ + /* Setup page metadata for small pages when page migration is enabled */ if (!pool->order && kbase_is_page_migration_enabled()) { INIT_LIST_HEAD(&p->lru); if (!kbase_alloc_page_metadata(kbdev, p, dma_addr, pool->group_id)) { @@ -398,13 +398,16 @@ int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow, pool->dont_reclaim = false; kbase_mem_pool_shrink_locked(pool, nr_to_grow); kbase_mem_pool_unlock(pool); + if (page_owner) + dev_info(pool->kbdev->dev, "%s : Ctx of process %s/%d dying", + __func__, page_owner->comm, task_pid_nr(page_owner)); - return -ENOMEM; + return -EPERM; } kbase_mem_pool_unlock(pool); if (unlikely(!can_alloc_page(pool, page_owner))) - return -ENOMEM; + return -EPERM; p = kbase_mem_alloc_page(pool, alloc_from_kthread); if (!p) { @@ -702,7 +705,7 @@ void kbase_mem_pool_free_locked(struct kbase_mem_pool *pool, struct page *p, boo } } -int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, +int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_small_pages, struct tagged_addr *pages, bool partial_allowed, struct task_struct *page_owner) { @@ -713,12 +716,12 @@ int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, size_t nr_pages_internal; const bool alloc_from_kthread = !!(current->flags & PF_KTHREAD); - nr_pages_internal = nr_4k_pages / (1u << (pool->order)); + nr_pages_internal = nr_small_pages / (1u << (pool->order)); - if (nr_pages_internal * (1u << pool->order) != nr_4k_pages) + if (nr_pages_internal * (1u << pool->order) != nr_small_pages) return -EINVAL; - pool_dbg(pool, "alloc_pages(4k=%zu):\n", nr_4k_pages); + pool_dbg(pool, "alloc_pages(small=%zu):\n", nr_small_pages); pool_dbg(pool, "alloc_pages(internal=%zu):\n", nr_pages_internal); /* Get pages from this pool */ @@ -741,18 +744,18 @@ int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_4k_pages, } kbase_mem_pool_unlock(pool); - if (i != nr_4k_pages && pool->next_pool) { + if (i != nr_small_pages && pool->next_pool) { /* Allocate via next pool */ - err = kbase_mem_pool_alloc_pages(pool->next_pool, nr_4k_pages - i, pages + i, + err = kbase_mem_pool_alloc_pages(pool->next_pool, nr_small_pages - i, pages + i, partial_allowed, page_owner); if (err < 0) goto err_rollback; - i += err; + i += (size_t)err; } else { /* Get any remaining pages from kernel */ - while (i != nr_4k_pages) { + while (i != nr_small_pages) { if (unlikely(!can_alloc_page(pool, page_owner))) goto err_rollback; @@ -789,7 +792,7 @@ err_rollback: return err; } -int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, size_t nr_4k_pages, +int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, size_t nr_small_pages, struct tagged_addr *pages) { struct page *p; @@ -798,12 +801,12 @@ int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, size_t nr_4k_ lockdep_assert_held(&pool->pool_lock); - nr_pages_internal = nr_4k_pages / (1u << (pool->order)); + nr_pages_internal = nr_small_pages / (1u << (pool->order)); - if (nr_pages_internal * (1u << pool->order) != nr_4k_pages) + if (nr_pages_internal * (1u << pool->order) != nr_small_pages) return -EINVAL; - pool_dbg(pool, "alloc_pages_locked(4k=%zu):\n", nr_4k_pages); + pool_dbg(pool, "alloc_pages_locked(small=%zu):\n", nr_small_pages); pool_dbg(pool, "alloc_pages_locked(internal=%zu):\n", nr_pages_internal); if (kbase_mem_pool_size(pool) < nr_pages_internal) { @@ -826,7 +829,7 @@ int kbase_mem_pool_alloc_pages_locked(struct kbase_mem_pool *pool, size_t nr_4k_ } } - return nr_4k_pages; + return nr_small_pages; } static void kbase_mem_pool_add_array(struct kbase_mem_pool *pool, size_t nr_pages, diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c index c1fcca6b47dc..1b224f7d0cc0 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.c @@ -29,7 +29,7 @@ void kbase_mem_pool_group_config_set_max_size(struct kbase_mem_pool_group_config size_t const max_size) { size_t const large_max_size = max_size >> (KBASE_MEM_POOL_2MB_PAGE_TABLE_ORDER - - KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER); + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER); int gid; for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { @@ -48,7 +48,7 @@ int kbase_mem_pool_group_init(struct kbase_mem_pool_group *const mem_pools, for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) { err = kbase_mem_pool_init(&mem_pools->small[gid], &configs->small[gid], - KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER, gid, kbdev, + KBASE_MEM_POOL_SMALL_PAGE_TABLE_ORDER, gid, kbdev, next_pools ? &next_pools->small[gid] : NULL); if (!err) { diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h index 27dd935c87cf..cb69eab061ac 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_pool_group.h @@ -53,7 +53,7 @@ kbase_mem_pool_group_select(struct kbase_device *kbdev, u32 mem_group_id, bool i * a set of memory pools * * @configs: Initial configuration for the set of memory pools - * @max_size: Maximum number of free 4 KiB pages each pool can hold + * @max_size: Maximum number of free small pages each pool can hold * * This function sets the initial configuration for every memory pool so that * the maximum amount of free memory that each pool can hold is identical. @@ -74,7 +74,7 @@ void kbase_mem_pool_group_config_set_max_size(struct kbase_mem_pool_group_config * * Initializes a complete set of physical memory pools. Memory pools are used to * allow efficient reallocation of previously-freed physical pages. A pair of - * memory pools is initialized for each physical memory group: one for 4 KiB + * memory pools is initialized for each physical memory group: one for small * pages and one for 2 MiB pages. * * If @next_pools is not NULL then a request to allocate memory from an diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h index d9729e3efbbb..56c7a4995ab5 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs.h @@ -30,6 +30,8 @@ #include #include +struct kbase_context; + /** * kbasep_mem_profile_debugfs_remove - Remove entry from Mali memory profile debugfs * diff --git a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c index e836011376b1..5e3d1eeb6d28 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c @@ -35,8 +35,12 @@ * @group_id: A physical memory group ID, which must be valid but is not used. * Its valid range is 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. * @gfp_mask: Bitmask of Get Free Page flags affecting allocator behavior. - * @order: Page order for physical page size (order=0 means 4 KiB, - * order=9 means 2 MiB). + * @order: Page order for physical page size. + * order = 0 refers to small pages + * order != 0 refers to 2 MB pages, so + * order = 9 (when small page size is 4KB, 2^9 * 4KB = 2 MB) + * order = 7 (when small page size is 16KB, 2^7 * 16KB = 2 MB) + * order = 5 (when small page size is 64KB, 2^5 * 64KB = 2 MB) * * Delegates all memory allocation requests to the kernel's alloc_pages * function. @@ -44,7 +48,8 @@ * Return: Pointer to allocated page, or NULL if allocation failed. */ static struct page *kbase_native_mgm_alloc(struct memory_group_manager_device *mgm_dev, - int group_id, gfp_t gfp_mask, unsigned int order) + unsigned int group_id, gfp_t gfp_mask, + unsigned int order) { /* * Check that the base and the mgm defines, from separate header files, @@ -78,8 +83,8 @@ static struct page *kbase_native_mgm_alloc(struct memory_group_manager_device *m * * Delegates all memory freeing requests to the kernel's __free_pages function. */ -static void kbase_native_mgm_free(struct memory_group_manager_device *mgm_dev, int group_id, - struct page *page, unsigned int order) +static void kbase_native_mgm_free(struct memory_group_manager_device *mgm_dev, + unsigned int group_id, struct page *page, unsigned int order) { CSTD_UNUSED(mgm_dev); CSTD_UNUSED(group_id); @@ -105,7 +110,8 @@ static void kbase_native_mgm_free(struct memory_group_manager_device *mgm_dev, i * entry was successfully installed. */ static vm_fault_t kbase_native_mgm_vmf_insert_pfn_prot(struct memory_group_manager_device *mgm_dev, - int group_id, struct vm_area_struct *vma, + unsigned int group_id, + struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot) { @@ -130,7 +136,7 @@ static vm_fault_t kbase_native_mgm_vmf_insert_pfn_prot(struct memory_group_manag * Return: A GPU page table entry to be stored in a page table. */ static u64 kbase_native_mgm_update_gpu_pte(struct memory_group_manager_device *mgm_dev, - int group_id, int mmu_level, u64 pte) + unsigned int group_id, int mmu_level, u64 pte) { CSTD_UNUSED(mgm_dev); CSTD_UNUSED(group_id); @@ -154,7 +160,7 @@ static u64 kbase_native_mgm_update_gpu_pte(struct memory_group_manager_device *m * Return: A GPU page table entry to be stored in a page table. */ static u64 kbase_native_mgm_pte_to_original_pte(struct memory_group_manager_device *mgm_dev, - int group_id, int mmu_level, u64 pte) + unsigned int group_id, int mmu_level, u64 pte) { CSTD_UNUSED(mgm_dev); CSTD_UNUSED(group_id); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pbha.c b/drivers/gpu/arm/bifrost/mali_kbase_pbha.c index 9f75c3371c15..341ea901e2e1 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pbha.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pbha.c @@ -155,7 +155,7 @@ int kbase_pbha_record_settings(struct kbase_device *kbdev, bool runtime, unsigne #if MALI_USE_CSF if (runtime) { - int i; + uint i; kbase_pm_context_active(kbdev); /* Ensure host copy of SYSC_ALLOC is up to date */ @@ -215,7 +215,7 @@ void kbase_pbha_write_settings(struct kbase_device *kbdev) { #if MALI_USE_CSF if (kbasep_pbha_supported(kbdev)) { - int i; + uint i; for (i = 0; i < GPU_SYSC_ALLOC_COUNT; ++i) kbase_reg_write32(kbdev, GPU_SYSC_ALLOC_OFFSET(i), kbdev->sysc_alloc[i]); @@ -249,9 +249,10 @@ static int kbase_pbha_read_int_id_override_property(struct kbase_device *kbdev, dev_err(kbdev->dev, "Bad DTB format: pbha.int_id_override\n"); return -EINVAL; } - if (of_property_read_u32_array(pbha_node, "int-id-override", dtb_data, sz) != 0) { + if (of_property_read_u32_array(pbha_node, "int-id-override", dtb_data, (size_t)sz) != 0) { /* There may be no int-id-override field. Fallback to int_id_override instead */ - if (of_property_read_u32_array(pbha_node, "int_id_override", dtb_data, sz) != 0) { + if (of_property_read_u32_array(pbha_node, "int_id_override", dtb_data, + (size_t)sz) != 0) { dev_err(kbdev->dev, "Failed to read DTB pbha.int_id_override\n"); return -EINVAL; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pbha.h b/drivers/gpu/arm/bifrost/mali_kbase_pbha.h index a8eb546a9a4d..9cecf83bf820 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pbha.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_pbha.h @@ -22,7 +22,9 @@ #ifndef _KBASE_PBHA_H #define _KBASE_PBHA_H -#include +#include + +struct kbase_device; /** * kbasep_pbha_supported - check whether PBHA registers are diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c index 5b13a0bd8f32..f1d2794dd86a 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c @@ -112,7 +112,7 @@ static ssize_t int_id_overrides_write(struct file *file, const char __user *ubuf kbase_pm_context_idle(kbdev); - return count; + return (ssize_t)count; } static int int_id_overrides_open(struct inode *in, struct file *file) @@ -196,7 +196,7 @@ static ssize_t propagate_bits_write(struct file *file, const char __user *ubuf, kbase_reset_gpu_wait(kbdev); } - return count; + return (ssize_t)count; } static const struct file_operations pbha_propagate_bits_fops = { diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.h b/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.h index 508ecdff9162..10b4c63abe74 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2021-2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -22,7 +22,7 @@ #ifndef _KBASE_PBHA_DEBUGFS_H #define _KBASE_PBHA_DEBUGFS_H -#include +struct kbase_device; /** * kbase_pbha_debugfs_init - Initialize pbha debugfs directory diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.c b/drivers/gpu/arm/bifrost/mali_kbase_pm.c index b636d4288511..ff71524eeaaa 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.c @@ -87,7 +87,7 @@ int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, } } c = ++kbdev->pm.active_count; - KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_ACTIVE, NULL, c); + KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_ACTIVE, NULL, (u64)c); if (c == 1) { /* First context active: Power on the GPU and @@ -117,7 +117,7 @@ void kbase_pm_context_idle(struct kbase_device *kbdev) kbase_pm_lock(kbdev); c = --kbdev->pm.active_count; - KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_IDLE, NULL, c); + KBASE_KTRACE_ADD(kbdev, PM_CONTEXT_IDLE, NULL, (u64)c); KBASE_DEBUG_ASSERT(c >= 0); @@ -200,8 +200,9 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) mutex_unlock(&kbdev->pm.lock); #ifdef CONFIG_MALI_ARBITER_SUPPORT +#if !MALI_USE_CSF if (kbdev->arb.arb_if) { - int i; + unsigned int i; unsigned long flags; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -211,6 +212,7 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) kbase_job_slot_softstop(kbdev, i, NULL); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } +#endif /* !MALI_USE_CSF */ #endif /* CONFIG_MALI_ARBITER_SUPPORT */ /* From now on, the active count will drop towards zero. Sometimes, diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.h b/drivers/gpu/arm/bifrost/mali_kbase_pm.h index 187fb9efaaf2..46db4db5ffe0 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pm.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.h @@ -26,7 +26,9 @@ #ifndef _KBASE_PM_H_ #define _KBASE_PM_H_ -#include "mali_kbase_hwaccess_pm.h" +#include + +struct kbase_device; #define PM_ENABLE_IRQS 0x01 #define PM_HW_ISSUES_DETECT 0x02 diff --git a/drivers/gpu/arm/bifrost/mali_kbase_refcount_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_refcount_defs.h deleted file mode 100644 index c517a2d2ab83..000000000000 --- a/drivers/gpu/arm/bifrost/mali_kbase_refcount_defs.h +++ /dev/null @@ -1,57 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef _KBASE_REFCOUNT_DEFS_H_ -#define _KBASE_REFCOUNT_DEFS_H_ - -/* - * The Refcount API is available from 4.11 onwards - * This file hides the compatibility issues with this for the rest the driver - */ - -#include -#include - -#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) - -#define kbase_refcount_t atomic_t -#define kbase_refcount_read(x) atomic_read(x) -#define kbase_refcount_set(x, v) atomic_set(x, v) -#define kbase_refcount_dec_and_test(x) atomic_dec_and_test(x) -#define kbase_refcount_dec(x) atomic_dec(x) -#define kbase_refcount_inc_not_zero(x) atomic_inc_not_zero(x) -#define kbase_refcount_inc(x) atomic_inc(x) - -#else - -#include - -#define kbase_refcount_t refcount_t -#define kbase_refcount_read(x) refcount_read(x) -#define kbase_refcount_set(x, v) refcount_set(x, v) -#define kbase_refcount_dec_and_test(x) refcount_dec_and_test(x) -#define kbase_refcount_dec(x) refcount_dec(x) -#define kbase_refcount_inc_not_zero(x) refcount_inc_not_zero(x) -#define kbase_refcount_inc(x) refcount_inc(x) - -#endif /* (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) */ - -#endif /* _KBASE_REFCOUNT_DEFS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c b/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c index e30857c7b35d..3128292a9a30 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c @@ -553,8 +553,8 @@ int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, { int err = 0; struct kbase_device *kbdev = kctx->kbdev; - const int cpu_va_bits = kbase_get_num_cpu_va_bits(kctx); - const int gpu_pc_bits = kbdev->gpu_props.log2_program_counter_size; + const size_t cpu_va_bits = kbase_get_num_cpu_va_bits(kctx); + const size_t gpu_pc_bits = kbdev->gpu_props.log2_program_counter_size; KBASE_DEBUG_ASSERT(kctx != NULL); KBASE_DEBUG_ASSERT(reg != NULL); @@ -1367,7 +1367,7 @@ struct kbase_va_region *kbase_alloc_free_region(struct kbase_reg_zone *zone, u64 return NULL; kbase_refcount_set(&new_reg->va_refcnt, 1); - atomic_set(&new_reg->no_user_free_count, 0); + atomic64_set(&new_reg->no_user_free_count, 0); new_reg->cpu_alloc = NULL; /* no alloc bound yet */ new_reg->gpu_alloc = NULL; /* no alloc bound yet */ new_reg->rbtree = &zone->reg_rbtree; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c index d3c030fe6915..14143f0bdb30 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_regs_history_debugfs.c @@ -133,7 +133,7 @@ void kbase_io_history_dump(struct kbase_device *kbdev) char const access = (io->addr & 1) ? 'w' : 'r'; dev_err(kbdev->dev, "%6zu: %c: reg 0x%16pK val %08x\n", i, access, - (void *)(io->addr & ~0x1), io->value); + (void *)(io->addr & ~(uintptr_t)0x1), io->value); } spin_unlock_irqrestore(&h->lock, flags); @@ -191,7 +191,7 @@ static int regs_history_show(struct seq_file *sfile, void *data) char const access = (io->addr & 1) ? 'w' : 'r'; seq_printf(sfile, "%6zu: %c: reg 0x%16pK val %08x\n", i, access, - (void *)(io->addr & ~0x1), io->value); + (void *)(io->addr & ~(uintptr_t)0x1), io->value); } spin_unlock_irqrestore(&h->lock, flags); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h b/drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h index 035b1b2bda3e..b7b2cf8cb2f4 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_reset_gpu.h @@ -22,6 +22,8 @@ #ifndef _KBASE_RESET_GPU_H_ #define _KBASE_RESET_GPU_H_ +struct kbase_device; + /** * kbase_reset_gpu_prevent_and_wait - Prevent GPU resets from starting whilst * the current thread is accessing the GPU, @@ -236,7 +238,7 @@ int kbase_reset_gpu_silent(struct kbase_device *kbdev); bool kbase_reset_gpu_is_active(struct kbase_device *kbdev); /** - * kbase_reset_gpu_not_pending - Reports if the GPU reset isn't pending + * kbase_reset_gpu_is_not_pending - Reports if the GPU reset isn't pending * * @kbdev: Device pointer * diff --git a/drivers/gpu/arm/bifrost/mali_kbase_smc.h b/drivers/gpu/arm/bifrost/mali_kbase_smc.h index b9f224f7ae51..7f298799a4ba 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_smc.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_smc.h @@ -24,13 +24,13 @@ #if IS_ENABLED(CONFIG_ARM64) -#include +#include -#define SMC_FAST_CALL (1 << 31) -#define SMC_64 (1 << 30) +#define SMC_FAST_CALL (1U << 31) +#define SMC_64 (1U << 30) #define SMC_OEN_OFFSET 24 -#define SMC_OEN_MASK (0x3F << SMC_OEN_OFFSET) /* 6 bits */ +#define SMC_OEN_MASK (0x3FU << SMC_OEN_OFFSET) /* 6 bits */ #define SMC_OEN_SIP (2 << SMC_OEN_OFFSET) #define SMC_OEN_STD (4 << SMC_OEN_OFFSET) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c index bbd756dbf840..0cee2f0e6fd5 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c @@ -83,7 +83,8 @@ static void kbasep_add_waiting_with_timeout(struct kbase_jd_atom *katom) /* Schedule timeout of this atom after a period if it is not active */ if (!timer_pending(&kctx->soft_job_timeout)) { - int timeout_ms = atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); + unsigned int timeout_ms = + (unsigned int)atomic_read(&kctx->kbdev->js_data.soft_job_timeout_ms); mod_timer(&kctx->soft_job_timeout, jiffies + msecs_to_jiffies(timeout_ms)); } } @@ -172,7 +173,7 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) kbase_pm_context_idle(kctx->kbdev); - data.sec = ts.tv_sec; + data.sec = (__u64)ts.tv_sec; data.usec = ts.tv_nsec / 1000; data.system_time = system_time; data.cycle_counter = cycle_counter; @@ -612,7 +613,7 @@ static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) /* Adjust number of pages, so that we only attempt to * release pages in the array that we know are valid. */ - buffers[i].nr_pages = pinned_pages; + buffers[i].nr_pages = (unsigned int)pinned_pages; ret = -EINVAL; goto out_cleanup; @@ -626,7 +627,8 @@ static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) kbase_gpu_vm_lock(katom->kctx); reg = kbase_region_tracker_find_region_enclosing_address( - katom->kctx, user_extres.ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE); + katom->kctx, + user_extres.ext_resource & ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE); if (kbase_is_region_invalid_or_free(reg) || reg->gpu_alloc == NULL) { ret = -EINVAL; @@ -668,7 +670,7 @@ static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) if (ret < 0) buffers[i].nr_extres_pages = 0; else - buffers[i].nr_extres_pages = ret; + buffers[i].nr_extres_pages = (unsigned int)ret; goto out_unlock; } @@ -1404,7 +1406,7 @@ static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) for (i = 0; i < ext_res->count; i++) { u64 gpu_addr; - gpu_addr = ext_res->ext_res[i].ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE; + gpu_addr = ext_res->ext_res[i].ext_resource & ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE; if (map) { if (!kbase_sticky_resource_acquire(katom->kctx, gpu_addr)) goto failed_loop; @@ -1431,7 +1433,7 @@ static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) failed_loop: while (i > 0) { u64 const gpu_addr = ext_res->ext_res[i - 1].ext_resource & - ~BASE_EXT_RES_ACCESS_EXCLUSIVE; + ~(__u64)BASE_EXT_RES_ACCESS_EXCLUSIVE; kbase_sticky_resource_release_force(katom->kctx, NULL, gpu_addr); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_sync.h b/drivers/gpu/arm/bifrost/mali_kbase_sync.h index ff5206d8d395..d871426f8a8a 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_sync.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_sync.h @@ -34,7 +34,12 @@ #include #endif -#include "mali_kbase.h" +#include + +#if !MALI_USE_CSF +struct kbase_jd_atom; +struct work_struct; +#endif /** * struct kbase_sync_fence_info - Information about a fence @@ -175,12 +180,8 @@ int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, struct kbase_sync #endif /* !MALI_USE_CSF */ #if IS_ENABLED(CONFIG_SYNC_FILE) -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -void kbase_sync_fence_info_get(struct fence *fence, struct kbase_sync_fence_info *info); -#else void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_info *info); #endif -#endif /** * kbase_sync_status_string() - Get string matching @status diff --git a/drivers/gpu/arm/bifrost/mali_kbase_sync_file.c b/drivers/gpu/arm/bifrost/mali_kbase_sync_file.c index aa4bf980e2bb..f0337e61a17e 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_sync_file.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_sync_file.c @@ -22,6 +22,9 @@ /* * Code for supporting explicit Linux fences (CONFIG_SYNC_FILE) */ +#include "mali_kbase_sync.h" +#include "mali_kbase_fence.h" +#include "mali_kbase.h" #include #include @@ -33,9 +36,7 @@ #include #include #include -#include "mali_kbase_sync.h" -#include "mali_kbase_fence.h" -#include "mali_kbase.h" +#include static const struct file_operations stream_fops = { .owner = THIS_MODULE }; @@ -54,11 +55,7 @@ int kbase_sync_fence_stream_create(const char *name, int *const out_fd) #if !MALI_USE_CSF int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; -#else struct dma_fence *fence; -#endif struct sync_file *sync_file; int fd; @@ -94,18 +91,14 @@ int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) return fd; } - fd_install(fd, sync_file->file); + fd_install((unsigned int)fd, sync_file->file); return fd; } int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence = sync_file_get_fence(fd); -#else struct dma_fence *fence = sync_file_get_fence(fd); -#endif lockdep_assert_held(&katom->kctx->jctx.lock); @@ -121,18 +114,14 @@ int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) int kbase_sync_fence_validate(int fd) { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence = sync_file_get_fence(fd); -#else struct dma_fence *fence = sync_file_get_fence(fd); -#endif if (!fence) return -EINVAL; dma_fence_put(fence); - return 0; /* valid */ + return 0; } #if !MALI_USE_CSF @@ -155,11 +144,7 @@ enum base_jd_event_code kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; } -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -static void kbase_fence_wait_callback(struct fence *fence, struct fence_cb *cb) -#else static void kbase_fence_wait_callback(struct dma_fence *fence, struct dma_fence_cb *cb) -#endif { struct kbase_jd_atom *katom = container_of(cb, struct kbase_jd_atom, dma_fence.fence_cb); struct kbase_context *kctx = katom->kctx; @@ -191,11 +176,7 @@ static void kbase_fence_wait_callback(struct dma_fence *fence, struct dma_fence_ int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) { int err; -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; -#else struct dma_fence *fence; -#endif lockdep_assert_held(&katom->kctx->jctx.lock); @@ -312,12 +293,9 @@ void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) } #endif /* !MALI_USE_CSF */ -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -void kbase_sync_fence_info_get(struct fence *fence, struct kbase_sync_fence_info *info) -#else void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_info *info) -#endif { + int status; info->fence = fence; /* Translate into the following status, with support for error handling: @@ -325,21 +303,14 @@ void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_ * 0 : active * 1 : signaled */ - if (dma_fence_is_signaled(fence)) { -#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ - (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ - KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) - int status = fence->error; -#else - int status = fence->status; -#endif - if (status < 0) - info->status = status; /* signaled with error */ - else - info->status = 1; /* signaled with success */ - } else { + status = dma_fence_get_status(fence); + + if (status < 0) + info->status = status; /* signaled with error */ + else if (status > 0) + info->status = 1; /* signaled with success */ + else info->status = 0; /* still active (unsignaled) */ - } #if (KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE) scnprintf(info->name, sizeof(info->name), "%llu#%u", fence->context, fence->seqno); @@ -351,11 +322,7 @@ void kbase_sync_fence_info_get(struct dma_fence *fence, struct kbase_sync_fence_ #if !MALI_USE_CSF int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info) { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; -#else struct dma_fence *fence; -#endif fence = kbase_fence_in_get(katom); if (!fence) @@ -370,11 +337,7 @@ int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, struct kbase_sync_ int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, struct kbase_sync_fence_info *info) { -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; -#else struct dma_fence *fence; -#endif fence = kbase_fence_out_get(katom); if (!fence) diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c index 8d6eb5fb651f..df027c727a2c 100644 --- a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c @@ -46,7 +46,7 @@ void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, struct kbase_mmu_setup (KBASE_MEMATTR_AARCH64_SHARED << (KBASE_MEMATTR_INDEX_SHARED * 8)); setup->transtab = (u64)mmut->pgd & AS_TRANSTAB_BASE_MASK; - setup->transcfg = AS_TRANSCFG_MODE_SET(0, AS_TRANSCFG_MODE_AARCH64_4K); + setup->transcfg = AS_TRANSCFG_MODE_SET(0ULL, AS_TRANSCFG_MODE_AARCH64_4K); } /** @@ -96,7 +96,7 @@ void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, struct u32 exception_type = fault->status & 0xFF; u32 access_type = (fault->status >> 8) & 0x3; u32 source_id = (fault->status >> 16); - int as_no; + u32 as_no; /* terminal fault, print info about the fault */ dev_err(kbdev->dev, @@ -112,7 +112,7 @@ void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, struct kbase_debug_csf_fault_notify(kbdev, NULL, DF_GPU_PAGE_FAULT); /* Report MMU fault for all address spaces (except MCU_AS_NR) */ - for (as_no = 1; as_no < kbdev->nr_hw_address_spaces; as_no++) + for (as_no = 1u; as_no < (u32)kbdev->nr_hw_address_spaces; as_no++) submit_work_pagefault(kbdev, as_no, fault); /* GPU reset is required to recover */ @@ -127,20 +127,20 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas { struct kbase_device *kbdev = kctx->kbdev; u32 const status = fault->status; - int exception_type = (status & GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) >> - GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT; - int access_type = (status & GPU_FAULTSTATUS_ACCESS_TYPE_MASK) >> - GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT; - int source_id = (status & GPU_FAULTSTATUS_SOURCE_ID_MASK) >> - GPU_FAULTSTATUS_SOURCE_ID_SHIFT; + unsigned int exception_type = (status & GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) >> + GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT; + unsigned int access_type = (status & GPU_FAULTSTATUS_ACCESS_TYPE_MASK) >> + GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT; + unsigned int source_id = (status & GPU_FAULTSTATUS_SOURCE_ID_MASK) >> + GPU_FAULTSTATUS_SOURCE_ID_SHIFT; const char *addr_valid = (status & GPU_FAULTSTATUS_ADDRESS_VALID_MASK) ? "true" : "false"; - int as_no = as->number; + unsigned int as_no = as->number; unsigned long flags; const uintptr_t fault_addr = fault->addr; /* terminal fault, print info about the fault */ dev_err(kbdev->dev, - "GPU bus fault in AS%d at PA %pK\n" + "GPU bus fault in AS%u at PA %pK\n" "PA_VALID: %s\n" "raw fault status: 0x%X\n" "exception type 0x%X: %s\n" @@ -152,12 +152,10 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas kbase_gpu_access_type_name(access_type), source_id, kctx->pid); /* AS transaction begin */ - mutex_lock(&kbdev->mmu_hw_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_mmu_disable(kctx); kbase_ctx_flag_set(kctx, KCTX_AS_DISABLED_ON_FAULT); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); /* Switching to UNMAPPED mode above would have enabled the firmware to * recover from the fault (if the memory access was made by firmware) @@ -182,40 +180,35 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as const char *reason_str, struct kbase_fault *fault) { unsigned long flags; - unsigned int exception_type; - unsigned int access_type; - unsigned int source_id; - int as_no; - struct kbase_device *kbdev; - const u32 status = fault->status; - - as_no = as->number; - kbdev = kctx->kbdev; + struct kbase_device *kbdev = kctx->kbdev; /* Make sure the context was active */ if (WARN_ON(atomic_read(&kctx->refcount) <= 0)) return; - /* decode the fault status */ - exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); - access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); - source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); + if (!kbase_ctx_flag(kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) { + const u32 status = fault->status; + /* decode the fault status */ + unsigned int exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); + unsigned int access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); + unsigned int source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); + unsigned int as_no = as->number; - /* terminal fault, print info about the fault */ - dev_err(kbdev->dev, - "Unhandled Page fault in AS%d at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X\n" - "pid: %d\n", - as_no, fault->addr, reason_str, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(status), source_id, kctx->pid); + /* terminal fault, print info about the fault */ + dev_err(kbdev->dev, + "Unhandled Page fault in AS%u at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X\n" + "pid: %d\n", + as_no, fault->addr, reason_str, status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(status), source_id, kctx->pid); + } /* AS transaction begin */ - mutex_lock(&kbdev->mmu_hw_mutex); /* switch to UNMAPPED mode, * will abort all jobs and stop any hw counter dumping @@ -227,7 +220,6 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as kbase_csf_ctx_report_page_fault_for_active_groups(kctx, fault); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->mmu_hw_mutex); /* AS transaction end */ /* Switching to UNMAPPED mode above would have enabled the firmware to @@ -252,7 +244,9 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as * @as: The address space that has the fault * @fault: Data relating to the fault * - * This function will process a fault on a specific address space + * This function will process a fault on a specific address space. + * The function must be called with the ref_count of the kctx already increased/acquired. + * If it fails to queue the work, the ref_count will be decreased. */ static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_as *as, struct kbase_fault *fault) @@ -291,11 +285,18 @@ static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase * We need to switch to UNMAPPED mode - but we do this in a * worker so that we can sleep */ - WARN_ON(!queue_work(as->pf_wq, &as->work_busfault)); - atomic_inc(&kbdev->faults_pending); + if (!queue_work(as->pf_wq, &as->work_busfault)) { + dev_warn(kbdev->dev, "Bus fault is already pending for as %u", as->number); + kbase_ctx_sched_release_ctx(kctx); + } else { + atomic_inc(&kbdev->faults_pending); + } } else { - WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); - atomic_inc(&kbdev->faults_pending); + if (!queue_work(as->pf_wq, &as->work_pagefault)) { + dev_warn(kbdev->dev, "Page fault is already pending for as %u", as->number); + kbase_ctx_sched_release_ctx(kctx); + } else + atomic_inc(&kbdev->faults_pending); } } @@ -350,7 +351,7 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) while (pf_bits) { struct kbase_context *kctx; - int as_no = ffs(pf_bits) - 1; + unsigned int as_no = (unsigned int)ffs((int)pf_bits) - 1; struct kbase_as *as = &kbdev->as[as_no]; struct kbase_fault *fault = &as->pf_data; @@ -510,7 +511,7 @@ void kbase_mmu_gpu_fault_interrupt(struct kbase_device *kbdev, u32 status, u32 a * the address space is invalid or it's MCU address space. */ for (as = 1; as < kbdev->nr_hw_address_spaces; as++) - submit_work_gpufault(kbdev, status, as, address); + submit_work_gpufault(kbdev, status, (u32)as, address); } else submit_work_gpufault(kbdev, status, as_nr, address); } diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c index 7cf0ed292fb0..1b2df11f3c3c 100644 --- a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c @@ -45,7 +45,7 @@ void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, struct kbase_mmu_setup (KBASE_MEMATTR_AARCH64_NON_CACHEABLE << (KBASE_MEMATTR_INDEX_NON_CACHEABLE * 8)); setup->transtab = (u64)mmut->pgd & AS_TRANSTAB_BASE_MASK; - setup->transcfg = AS_TRANSCFG_MODE_SET(0, AS_TRANSCFG_MODE_AARCH64_4K); + setup->transcfg = AS_TRANSCFG_MODE_SET(0ULL, AS_TRANSCFG_MODE_AARCH64_4K); } void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbase_as *as, @@ -55,13 +55,13 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas u32 const status = fault->status; u32 const exception_type = (status & 0xFF); u32 const exception_data = (status >> 8) & 0xFFFFFF; - int const as_no = as->number; + unsigned int const as_no = as->number; unsigned long flags; const uintptr_t fault_addr = fault->addr; /* terminal fault, print info about the fault */ dev_err(kbdev->dev, - "GPU bus fault in AS%d at PA %pK\n" + "GPU bus fault in AS%u at PA %pK\n" "raw fault status: 0x%X\n" "exception type 0x%X: %s\n" "exception data 0x%X\n" @@ -95,38 +95,33 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as const char *reason_str, struct kbase_fault *fault) { unsigned long flags; - u32 exception_type; - u32 access_type; - u32 source_id; - int as_no; - struct kbase_device *kbdev; - struct kbasep_js_device_data *js_devdata; - - as_no = as->number; - kbdev = kctx->kbdev; - js_devdata = &kbdev->js_data; + struct kbase_device *kbdev = kctx->kbdev; + struct kbasep_js_device_data *js_devdata = &kbdev->js_data; + unsigned int as_no = as->number; /* Make sure the context was active */ if (WARN_ON(atomic_read(&kctx->refcount) <= 0)) return; - /* decode the fault status */ - exception_type = fault->status & 0xFF; - access_type = (fault->status >> 8) & 0x3; - source_id = (fault->status >> 16); + if (!kbase_ctx_flag(kctx, KCTX_PAGE_FAULT_REPORT_SKIP)) { + /* decode the fault status */ + u32 exception_type = fault->status & 0xFF; + u32 access_type = (fault->status >> 8) & 0x3; + u32 source_id = (fault->status >> 16); - /* terminal fault, print info about the fault */ - dev_err(kbdev->dev, - "Unhandled Page fault in AS%d at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X\n" - "pid: %d\n", - as_no, fault->addr, reason_str, fault->status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(fault->status), source_id, kctx->pid); + /* terminal fault, print info about the fault */ + dev_err(kbdev->dev, + "Unhandled Page fault in AS%u at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X\n" + "pid: %d\n", + as_no, fault->addr, reason_str, fault->status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(fault->status), source_id, kctx->pid); + } /* hardware counters dump fault handling */ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); @@ -193,12 +188,12 @@ static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase if (kbase_as_has_bus_fault(as, fault)) { dev_warn( kbdev->dev, - "Bus error in AS%d at PA 0x%pK with no context present! Spurious IRQ or SW Design Error?\n", + "Bus error in AS%u at PA 0x%pK with no context present! Spurious IRQ or SW Design Error?\n", as->number, (void *)(uintptr_t)fault->addr); } else { dev_warn( kbdev->dev, - "Page fault in AS%d at VA 0x%016llx with no context present! Spurious IRQ or SW Design Error?\n", + "Page fault in AS%u at VA 0x%016llx with no context present! Spurious IRQ or SW Design Error?\n", as->number, fault->addr); } /* Since no ctx was found, the MMU must be disabled. */ @@ -236,7 +231,7 @@ static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase */ kbasep_js_clear_submit_allowed(js_devdata, kctx); - dev_warn(kbdev->dev, "Bus error in AS%d at PA=0x%pK, IPA=0x%pK\n", as->number, + dev_warn(kbdev->dev, "Bus error in AS%u at PA=0x%pK, IPA=0x%pK\n", as->number, (void *)(uintptr_t)fault->addr, (void *)(uintptr_t)fault->extra_addr); /* @@ -310,11 +305,11 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) * the while logic ensures we have a bit set, no need to check * for not-found here */ - as_no = ffs(bf_bits | pf_bits) - 1; + as_no = (unsigned int)ffs((int)(bf_bits | pf_bits)) - 1; as = &kbdev->as[as_no]; /* find the fault type */ - if (bf_bits & (1 << as_no)) + if (bf_bits & (1UL << as_no)) fault = &as->bf_data; else fault = &as->pf_data; diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c index f3095f3b1f2a..becbb02aa15a 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c @@ -74,54 +74,6 @@ static void release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) #endif /* MALI_USE_CSF */ } -static void mmu_hw_operation_begin(struct kbase_device *kbdev) -{ -#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) -#if MALI_USE_CSF - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2019_3878)) { - unsigned long flags; - - lockdep_assert_held(&kbdev->mmu_hw_mutex); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - WARN_ON_ONCE(kbdev->mmu_hw_operation_in_progress); - kbdev->mmu_hw_operation_in_progress = true; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } -#else - CSTD_UNUSED(kbdev); -#endif /* MALI_USE_CSF */ -#else - CSTD_UNUSED(kbdev); -#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ -} - -static void mmu_hw_operation_end(struct kbase_device *kbdev) -{ -#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) -#if MALI_USE_CSF - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2019_3878)) { - unsigned long flags; - - lockdep_assert_held(&kbdev->mmu_hw_mutex); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - WARN_ON_ONCE(!kbdev->mmu_hw_operation_in_progress); - kbdev->mmu_hw_operation_in_progress = false; - /* Invoke the PM state machine, the L2 power off may have been - * skipped due to the MMU command. - */ - kbase_pm_update_state(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } -#else - CSTD_UNUSED(kbdev); -#endif /* MALI_USE_CSF */ -#else - CSTD_UNUSED(kbdev); -#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ -} - /** * mmu_flush_cache_on_gpu_ctrl() - Check if cache flush needs to be done * through GPU_CONTROL interface. @@ -210,7 +162,7 @@ static void mmu_flush_invalidate_as(struct kbase_device *kbdev, struct kbase_as mutex_lock(&kbdev->mmu_hw_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (kbdev->pm.backend.gpu_ready && (kbase_mmu_hw_do_flush_locked(kbdev, as, op_param))) + if (kbdev->pm.backend.gpu_ready && kbase_mmu_hw_do_flush(kbdev, as, op_param)) dev_err(kbdev->dev, "Flush for GPU page table update did not complete"); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -592,10 +544,9 @@ static void kbase_gpu_mmu_handle_write_faulting_as(struct kbase_device *kbdev, */ const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_SYNC; struct kbase_mmu_hw_op_param op_param; + unsigned long irq_flags; int ret = 0; - mutex_lock(&kbdev->mmu_hw_mutex); - kbase_mmu_hw_clear_fault(kbdev, faulting_as, KBASE_MMU_FAULT_TYPE_PAGE); /* flush L2 and unlock the VA (resumes the MMU) */ @@ -604,20 +555,14 @@ static void kbase_gpu_mmu_handle_write_faulting_as(struct kbase_device *kbdev, op_param.op = KBASE_MMU_OP_FLUSH_PT; op_param.kctx_id = kctx_id; op_param.mmu_sync_info = mmu_sync_info; + spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); if (mmu_flush_cache_on_gpu_ctrl(kbdev)) { - unsigned long irq_flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); op_param.flush_skip_levels = pgd_level_to_skip_flush(dirty_pgds); ret = kbase_mmu_hw_do_flush_on_gpu_ctrl(kbdev, faulting_as, &op_param); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); } else { - mmu_hw_operation_begin(kbdev); ret = kbase_mmu_hw_do_flush(kbdev, faulting_as, &op_param); - mmu_hw_operation_end(kbdev); } - - mutex_unlock(&kbdev->mmu_hw_mutex); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); if (ret) dev_err(kbdev->dev, @@ -630,14 +575,14 @@ static void set_gwt_element_page_addr_and_size(struct kbasep_gwt_list_element *e u64 fault_page_addr, struct tagged_addr fault_phys) { u64 fault_pfn = fault_page_addr >> PAGE_SHIFT; - unsigned int vindex = fault_pfn & (NUM_4K_PAGES_IN_2MB_PAGE - 1); + unsigned int vindex = fault_pfn & (NUM_PAGES_IN_2MB_LARGE_PAGE - 1); /* If the fault address lies within a 2MB page, then consider * the whole 2MB page for dumping to avoid incomplete dumps. */ if (is_huge(fault_phys) && (vindex == index_in_large_page(fault_phys))) { - element->page_addr = fault_page_addr & ~(SZ_2M - 1); - element->num_pages = NUM_4K_PAGES_IN_2MB_PAGE; + element->page_addr = fault_page_addr & ~(SZ_2M - 1UL); + element->num_pages = NUM_PAGES_IN_2MB_LARGE_PAGE; } else { element->page_addr = fault_page_addr; element->num_pages = 1; @@ -653,7 +598,7 @@ static void kbase_gpu_mmu_handle_write_fault(struct kbase_context *kctx, struct tagged_addr *fault_phys_addr; struct kbase_fault *fault; u64 fault_pfn, pfn_offset; - int as_no; + unsigned int as_no; u64 dirty_pgds = 0; as_no = faulting_as->number; @@ -750,7 +695,7 @@ static void kbase_gpu_mmu_handle_permission_fault(struct kbase_context *kctx, * estimate_pool_space_required - Determine how much a pool should be grown by to support a future * allocation * @pool: The memory pool to check, including its linked pools - * @pages_required: Number of 4KiB pages require for the pool to support a future allocation + * @pages_required: Number of small pages require for the pool to support a future allocation * * The value returned is accounting for the size of @pool and the size of each memory pool linked to * @pool. Hence, the caller should use @pool and (if not already satisfied) all its linked pools to @@ -762,7 +707,7 @@ static void kbase_gpu_mmu_handle_permission_fault(struct kbase_context *kctx, * should keep attempting an allocation and then re-growing with a new value queried form this * function until the allocation succeeds. * - * Return: an estimate of the amount of extra 4KiB pages in @pool that are required to satisfy an + * Return: an estimate of the amount of extra small pages in @pool that are required to satisfy an * allocation, or 0 if @pool (including its linked pools) is likely to already satisfy the * allocation. */ @@ -772,15 +717,15 @@ static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const si for (pages_still_required = pages_required; pool != NULL && pages_still_required; pool = pool->next_pool) { - size_t pool_size_4k; + size_t pool_size_small; kbase_mem_pool_lock(pool); - pool_size_4k = kbase_mem_pool_size(pool) << pool->order; - if (pool_size_4k >= pages_still_required) + pool_size_small = kbase_mem_pool_size(pool) << pool->order; + if (pool_size_small >= pages_still_required) pages_still_required = 0; else - pages_still_required -= pool_size_4k; + pages_still_required -= pool_size_small; kbase_mem_pool_unlock(pool); } @@ -791,11 +736,11 @@ static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const si * page_fault_try_alloc - Try to allocate memory from a context pool * @kctx: Context pointer * @region: Region to grow - * @new_pages: Number of 4 KiB pages to allocate + * @new_pages: Number of small pages to allocate * @pages_to_grow: Pointer to variable to store number of outstanding pages on failure. This can be - * either 4 KiB or 2 MiB pages, depending on the number of pages requested. + * either small or 2 MiB pages, depending on the number of pages requested. * @grow_2mb_pool: Pointer to variable to store which pool needs to grow - true for 2 MiB, false for - * 4 KiB. + * pool of small pages. * @prealloc_sas: Pointer to kbase_sub_alloc structures * * This function will try to allocate as many pages as possible from the context pool, then if @@ -807,12 +752,12 @@ static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const si * held could invoke the OoM killer and cause an effective deadlock with kbase_cpu_vm_close(). * * If 2 MiB pages are enabled and new_pages is >= 2 MiB then pages_to_grow will be a count of 2 MiB - * pages, otherwise it will be a count of 4 KiB pages. + * pages, otherwise it will be a count of small pages. * * Return: true if successful, false on failure */ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_region *region, - size_t new_pages, int *pages_to_grow, bool *grow_2mb_pool, + size_t new_pages, size_t *pages_to_grow, bool *grow_2mb_pool, struct kbase_sub_alloc **prealloc_sas) { size_t total_gpu_pages_alloced = 0; @@ -820,7 +765,7 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg struct kbase_mem_pool *pool, *root_pool; bool alloc_failed = false; size_t pages_still_required; - size_t total_mempools_free_4k = 0; + size_t total_mempools_free_small = 0; lockdep_assert_held(&kctx->reg_lock); lockdep_assert_held(&kctx->mem_partials_lock); @@ -831,7 +776,7 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg return false; } - if (kctx->kbdev->pagesize_2mb && new_pages >= (SZ_2M / SZ_4K)) { + if (kctx->kbdev->pagesize_2mb && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE) { root_pool = &kctx->mem_pools.large[region->gpu_alloc->group_id]; *grow_2mb_pool = true; } else { @@ -861,40 +806,40 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg */ pages_still_required = new_pages; for (pool = root_pool; pool != NULL && pages_still_required; pool = pool->next_pool) { - size_t pool_size_4k; - size_t pages_to_alloc_4k; - size_t pages_to_alloc_4k_per_alloc; + size_t pool_size_small; + size_t pages_to_alloc_small; + size_t pages_to_alloc_small_per_alloc; kbase_mem_pool_lock(pool); /* Allocate as much as possible from this pool*/ - pool_size_4k = kbase_mem_pool_size(pool) << pool->order; - total_mempools_free_4k += pool_size_4k; - pages_to_alloc_4k = MIN(pages_still_required, pool_size_4k); + pool_size_small = kbase_mem_pool_size(pool) << pool->order; + total_mempools_free_small += pool_size_small; + pages_to_alloc_small = MIN(pages_still_required, pool_size_small); if (region->gpu_alloc == region->cpu_alloc) - pages_to_alloc_4k_per_alloc = pages_to_alloc_4k; + pages_to_alloc_small_per_alloc = pages_to_alloc_small; else - pages_to_alloc_4k_per_alloc = pages_to_alloc_4k >> 1; + pages_to_alloc_small_per_alloc = pages_to_alloc_small >> 1; - if (pages_to_alloc_4k) { + if (pages_to_alloc_small) { struct tagged_addr *gpu_pages = kbase_alloc_phy_pages_helper_locked( - region->gpu_alloc, pool, pages_to_alloc_4k_per_alloc, + region->gpu_alloc, pool, pages_to_alloc_small_per_alloc, &prealloc_sas[0]); if (!gpu_pages) alloc_failed = true; else - total_gpu_pages_alloced += pages_to_alloc_4k_per_alloc; + total_gpu_pages_alloced += pages_to_alloc_small_per_alloc; if (!alloc_failed && region->gpu_alloc != region->cpu_alloc) { struct tagged_addr *cpu_pages = kbase_alloc_phy_pages_helper_locked( - region->cpu_alloc, pool, pages_to_alloc_4k_per_alloc, + region->cpu_alloc, pool, pages_to_alloc_small_per_alloc, &prealloc_sas[1]); if (!cpu_pages) alloc_failed = true; else - total_cpu_pages_alloced += pages_to_alloc_4k_per_alloc; + total_cpu_pages_alloced += pages_to_alloc_small_per_alloc; } } @@ -902,12 +847,12 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg if (alloc_failed) { WARN_ON(!pages_still_required); - WARN_ON(pages_to_alloc_4k >= pages_still_required); - WARN_ON(pages_to_alloc_4k_per_alloc >= pages_still_required); + WARN_ON(pages_to_alloc_small >= pages_still_required); + WARN_ON(pages_to_alloc_small_per_alloc >= pages_still_required); break; } - pages_still_required -= pages_to_alloc_4k; + pages_still_required -= pages_to_alloc_small; } if (pages_still_required) { @@ -931,7 +876,7 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg kctx->kbdev->dev, "Page allocation failure of %zu pages: managed %zu pages, mempool (inc linked pools) had %zu pages available", new_pages, total_gpu_pages_alloced + total_cpu_pages_alloced, - total_mempools_free_4k); + total_mempools_free_small); *pages_to_grow = 0; } else { /* Tell the caller to try to grow the memory pool @@ -970,14 +915,14 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) size_t new_pages; size_t fault_rel_pfn; struct kbase_as *faulting_as; - int as_no; + unsigned int as_no; struct kbase_context *kctx; struct kbase_device *kbdev; struct kbase_va_region *region; struct kbase_fault *fault; int err; bool grown = false; - int pages_to_grow; + size_t pages_to_grow; bool grow_2mb_pool; struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; int i; @@ -985,6 +930,7 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) #if MALI_JIT_PRESSURE_LIMIT_BASE size_t pages_trimmed = 0; #endif + unsigned long hwaccess_flags; /* Calls to this function are inherently synchronous, with respect to * MMU operations. @@ -997,7 +943,7 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) as_no = faulting_as->number; kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); - dev_dbg(kbdev->dev, "Entering %s %pK, fault_pfn %lld, as_no %d", __func__, (void *)data, + dev_dbg(kbdev->dev, "Entering %s %pK, fault_pfn %lld, as_no %u", __func__, (void *)data, fault_pfn, as_no); /* Grab the context that was already refcounted in kbase_mmu_interrupt() @@ -1223,8 +1169,6 @@ page_fault_retry: "Page fault @ VA 0x%llx in allocated region 0x%llx-0x%llx of growable TMEM: Ignoring", fault->addr, region->start_pfn, region->start_pfn + current_backed_size); - mutex_lock(&kbdev->mmu_hw_mutex); - kbase_mmu_hw_clear_fault(kbdev, faulting_as, KBASE_MMU_FAULT_TYPE_PAGE); /* [1] in case another page fault occurred while we were * handling the (duplicate) page fault we need to ensure we @@ -1236,19 +1180,15 @@ page_fault_retry: */ op_param.mmu_sync_info = mmu_sync_info; op_param.kctx_id = kctx->id; - if (!mmu_flush_cache_on_gpu_ctrl(kbdev)) { - mmu_hw_operation_begin(kbdev); - err = kbase_mmu_hw_do_unlock_no_addr(kbdev, faulting_as, &op_param); - mmu_hw_operation_end(kbdev); - } else { - /* Can safely skip the invalidate for all levels in case - * of duplicate page faults. - */ - op_param.flush_skip_levels = 0xF; - op_param.vpfn = fault_pfn; - op_param.nr = 1; - err = kbase_mmu_hw_do_unlock(kbdev, faulting_as, &op_param); - } + /* Can safely skip the invalidate for all levels in case + * of duplicate page faults. + */ + op_param.flush_skip_levels = 0xF; + op_param.vpfn = fault_pfn; + op_param.nr = 1; + spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_flags); + err = kbase_mmu_hw_do_unlock(kbdev, faulting_as, &op_param); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, hwaccess_flags); if (err) { dev_err(kbdev->dev, @@ -1256,8 +1196,6 @@ page_fault_retry: fault->addr); } - mutex_unlock(&kbdev->mmu_hw_mutex); - kbase_mmu_hw_enable_fault(kbdev, faulting_as, KBASE_MMU_FAULT_TYPE_PAGE); kbase_gpu_vm_unlock(kctx); @@ -1273,27 +1211,21 @@ page_fault_retry: if (new_pages == 0) { struct kbase_mmu_hw_op_param op_param; - mutex_lock(&kbdev->mmu_hw_mutex); - /* Duplicate of a fault we've already handled, nothing to do */ kbase_mmu_hw_clear_fault(kbdev, faulting_as, KBASE_MMU_FAULT_TYPE_PAGE); /* See comment [1] about UNLOCK usage */ op_param.mmu_sync_info = mmu_sync_info; op_param.kctx_id = kctx->id; - if (!mmu_flush_cache_on_gpu_ctrl(kbdev)) { - mmu_hw_operation_begin(kbdev); - err = kbase_mmu_hw_do_unlock_no_addr(kbdev, faulting_as, &op_param); - mmu_hw_operation_end(kbdev); - } else { - /* Can safely skip the invalidate for all levels in case - * of duplicate page faults. - */ - op_param.flush_skip_levels = 0xF; - op_param.vpfn = fault_pfn; - op_param.nr = 1; - err = kbase_mmu_hw_do_unlock(kbdev, faulting_as, &op_param); - } + /* Can safely skip the invalidate for all levels in case + * of duplicate page faults. + */ + op_param.flush_skip_levels = 0xF; + op_param.vpfn = fault_pfn; + op_param.nr = 1; + spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_flags); + err = kbase_mmu_hw_do_unlock(kbdev, faulting_as, &op_param); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, hwaccess_flags); if (err) { dev_err(kbdev->dev, @@ -1301,8 +1233,6 @@ page_fault_retry: fault->addr); } - mutex_unlock(&kbdev->mmu_hw_mutex); - kbase_mmu_hw_enable_fault(kbdev, faulting_as, KBASE_MMU_FAULT_TYPE_PAGE); kbase_gpu_vm_unlock(kctx); goto fault_done; @@ -1380,7 +1310,6 @@ page_fault_retry: #endif /* AS transaction begin */ - mutex_lock(&kbdev->mmu_hw_mutex); /* clear MMU interrupt - this needs to be done after updating * the page tables but before issuing a FLUSH command. The @@ -1397,16 +1326,16 @@ page_fault_retry: op_param.op = KBASE_MMU_OP_FLUSH_PT; op_param.kctx_id = kctx->id; op_param.mmu_sync_info = mmu_sync_info; + spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_flags); if (mmu_flush_cache_on_gpu_ctrl(kbdev)) { /* Unlock to invalidate the TLB (and resume the MMU) */ op_param.flush_skip_levels = pgd_level_to_skip_flush(dirty_pgds); err = kbase_mmu_hw_do_unlock(kbdev, faulting_as, &op_param); } else { /* flush L2 and unlock the VA (resumes the MMU) */ - mmu_hw_operation_begin(kbdev); err = kbase_mmu_hw_do_flush(kbdev, faulting_as, &op_param); - mmu_hw_operation_end(kbdev); } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, hwaccess_flags); if (err) { dev_err(kbdev->dev, @@ -1414,7 +1343,6 @@ page_fault_retry: fault->addr); } - mutex_unlock(&kbdev->mmu_hw_mutex); /* AS transaction end */ /* reenable this in the mask */ @@ -1459,8 +1387,9 @@ page_fault_retry: struct kbase_mem_pool *const lp_mem_pool = &kctx->mem_pools.large[group_id]; - pages_to_grow = (pages_to_grow + ((1 << lp_mem_pool->order) - 1)) >> - lp_mem_pool->order; + pages_to_grow = + (pages_to_grow + ((1u << lp_mem_pool->order) - 1u)) >> + lp_mem_pool->order; ret = kbase_mem_pool_grow(lp_mem_pool, pages_to_grow, kctx->task); } else { @@ -1472,6 +1401,8 @@ page_fault_retry: } if (ret < 0) { /* failed to extend, handle as a normal PF */ + if (unlikely(ret == -EPERM)) + kbase_ctx_flag_set(kctx, KCTX_PAGE_FAULT_REPORT_SKIP); kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Page allocation failure", fault); } else { @@ -1560,7 +1491,7 @@ alloc_free: * @kbdev: Device pointer. * @mmut: GPU MMU page table. * @pgd: Physical addresse of level N page directory. - * @vpfn: The virtual page frame number. + * @vpfn: The virtual page frame number, in GPU_PAGE_SIZE units. * @level: The level of MMU page table (N). * * Return: @@ -1613,7 +1544,7 @@ static int mmu_get_next_pgd(struct kbase_device *kbdev, struct kbase_mmu_table * * * @kbdev: Device pointer. * @mmut: GPU MMU page table. - * @vpfn: The virtual page frame number. + * @vpfn: The virtual page frame number, in GPU_PAGE_SIZE units. * @in_level: The level of MMU page table (N). * @out_level: Set to the level of the lowest valid PGD found on success. * Invalid on error. @@ -1702,8 +1633,10 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, u64 vpfn = from_vpfn; struct kbase_mmu_mode const *mmu_mode; + /* Both from_vpfn and to_vpfn are in GPU_PAGE_SIZE units */ + /* 64-bit address range is the max */ - KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); + KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / GPU_PAGE_SIZE)); KBASE_DEBUG_ASSERT(from_vpfn <= to_vpfn); lockdep_assert_held(&mmut->mmu_lock); @@ -1727,7 +1660,7 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, if (count > left) count = left; - /* need to check if this is a 2MB page or a 4kB */ + /* need to check if this is a 2MB page or a small page */ for (level = MIDGARD_MMU_TOPLEVEL; level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { idx = (vpfn >> ((3 - level) * 9)) & 0x1FF; pgds[level] = pgd; @@ -1796,7 +1729,7 @@ next: * movable once they are returned to a memory pool. */ if (kbase_is_page_migration_enabled() && !ignore_page_migration && phys) { - const u64 num_pages = to_vpfn - from_vpfn; + const u64 num_pages = (to_vpfn - from_vpfn) / GPU_PAGES_PER_CPU_PAGE; u64 i; for (i = 0; i < num_pages; i++) { @@ -1862,7 +1795,7 @@ static void mmu_flush_invalidate_insert_pages(struct kbase_device *kbdev, * The bottom PGD level. * @insert_level: The level of MMU page table where the chain of newly allocated * PGDs needs to be linked-in/inserted. - * @insert_vpfn: The virtual page frame number for the ATE. + * @insert_vpfn: The virtual page frame number, in GPU_PAGE_SIZE units, for the ATE. * @pgds_to_insert: Ptr to an array (size MIDGARD_MMU_BOTTOMLEVEL+1) that contains * the physical addresses of newly allocated PGDs from index * insert_level+1 to cur_level, and an existing PGD at index @@ -2010,7 +1943,7 @@ static int mmu_insert_alloc_pgds(struct kbase_device *kbdev, struct kbase_mmu_ta mutex_unlock(&mmut->mmu_lock); err = kbase_mem_pool_grow(&kbdev->mem_pools.small[mmut->group_id], - level_high, NULL); + (size_t)level_high, NULL); mutex_lock(&mmut->mmu_lock); if (err) { dev_err(kbdev->dev, "%s: kbase_mem_pool_grow() returned error %d", @@ -2050,6 +1983,7 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp enum kbase_mmu_op_type flush_op; struct kbase_mmu_table *mmut = &kctx->mmu; int l, cur_level, insert_level; + const phys_addr_t base_phys_address = as_phys_addr_t(phys); if (WARN_ON(kctx == NULL)) return -EINVAL; @@ -2063,6 +1997,10 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp if (nr == 0) return 0; + /* Convert to GPU_PAGE_SIZE units. */ + insert_vpfn *= GPU_PAGES_PER_CPU_PAGE; + remain *= GPU_PAGES_PER_CPU_PAGE; + /* If page migration is enabled, pages involved in multiple GPU mappings * are always treated as not movable. */ @@ -2142,14 +2080,19 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp num_of_valid_entries = kbdev->mmu_mode->get_num_valid_entries(pgd_page); - for (i = 0; i < count; i++) { - unsigned int ofs = vindex + i; + for (i = 0; i < count; i += GPU_PAGES_PER_CPU_PAGE) { + unsigned int j; - /* Fail if the current page is a valid ATE entry */ - KBASE_DEBUG_ASSERT(0 == (pgd_page[ofs] & 1UL)); + for (j = 0; j < GPU_PAGES_PER_CPU_PAGE; j++) { + unsigned int ofs = vindex + i + j; + phys_addr_t page_address = base_phys_address + (j * GPU_PAGE_SIZE); - pgd_page[ofs] = kbase_mmu_create_ate(kbdev, phys, flags, - MIDGARD_MMU_BOTTOMLEVEL, group_id); + /* Fail if the current page is a valid ATE entry */ + WARN_ON_ONCE((pgd_page[ofs] & 1UL)); + pgd_page[ofs] = kbase_mmu_create_ate(kbdev, as_tagged(page_address), + flags, MIDGARD_MMU_BOTTOMLEVEL, + group_id); + } } kbdev->mmu_mode->set_num_valid_entries(pgd_page, num_of_valid_entries + count); @@ -2201,10 +2144,10 @@ fail_unlock_free_pgds: kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]); fail_unlock: - if (insert_vpfn != start_vpfn) { + if (insert_vpfn != (start_vpfn * GPU_PAGES_PER_CPU_PAGE)) { /* Invalidate the pages we have partially completed */ - mmu_insert_pages_failure_recovery(kbdev, mmut, start_vpfn, insert_vpfn, &dirty_pgds, - NULL, true); + mmu_insert_pages_failure_recovery(kbdev, mmut, start_vpfn * GPU_PAGES_PER_CPU_PAGE, + insert_vpfn, &dirty_pgds, NULL, true); } mmu_flush_invalidate_insert_pages(kbdev, mmut, start_vpfn, nr, dirty_pgds, mmu_sync_info, @@ -2286,7 +2229,7 @@ static void kbase_mmu_progress_migration_on_teardown(struct kbase_device *kbdev, struct page *phys_page = as_page(phys[i]); struct kbase_page_metadata *page_md = kbase_page_private(phys_page); - /* Skip the 4KB page that is part of a large page, as the large page is + /* Skip the small page that is part of a large page, as the large page is * excluded from the migration process. */ if (is_huge(phys[i]) || is_partial(phys[i])) @@ -2326,11 +2269,12 @@ u64 kbase_mmu_create_ate(struct kbase_device *const kbdev, struct tagged_addr co u64 entry; kbdev->mmu_mode->entry_set_ate(&entry, phy, flags, level); - return kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, group_id, level, entry); + return kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, (unsigned int)group_id, level, + entry); } static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, - const u64 start_vpfn, struct tagged_addr *phys, size_t nr, + u64 start_vpfn, struct tagged_addr *phys, size_t nr, unsigned long flags, int const group_id, u64 *dirty_pgds, struct kbase_va_region *reg, bool ignore_page_migration) { @@ -2355,6 +2299,9 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm if (nr == 0) return 0; + /* Convert to GPU_PAGE_SIZE units. */ + insert_vpfn *= GPU_PAGES_PER_CPU_PAGE; + remain *= GPU_PAGES_PER_CPU_PAGE; mutex_lock(&mmut->mmu_lock); while (remain) { @@ -2432,29 +2379,39 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm num_of_valid_entries++; } else { - for (i = 0; i < count; i++) { - unsigned int ofs = vindex + i; - u64 *target = &pgd_page[ofs]; + for (i = 0; i < count; i += GPU_PAGES_PER_CPU_PAGE) { + struct tagged_addr base_tagged_addr = + phys[i / GPU_PAGES_PER_CPU_PAGE]; + phys_addr_t base_phys_address = as_phys_addr_t(base_tagged_addr); + unsigned int j; - /* Warn if the current page is a valid ATE - * entry. The page table shouldn't have anything - * in the place where we are trying to put a - * new entry. Modification to page table entries - * should be performed with - * kbase_mmu_update_pages() - */ - WARN_ON((*target & 1UL) != 0); + for (j = 0; j < GPU_PAGES_PER_CPU_PAGE; j++) { + unsigned int ofs = vindex + i + j; + u64 *target = &pgd_page[ofs]; + phys_addr_t page_address = + base_phys_address + (j * GPU_PAGE_SIZE); - *target = kbase_mmu_create_ate(kbdev, phys[i], flags, cur_level, - group_id); + /* Warn if the current page is a valid ATE + * entry. The page table shouldn't have anything + * in the place where we are trying to put a + * new entry. Modification to page table entries + * should be performed with + * kbase_mmu_update_pages() + */ + WARN_ON_ONCE((*target & 1UL) != 0); + + *target = kbase_mmu_create_ate(kbdev, + as_tagged(page_address), + flags, cur_level, group_id); + } /* If page migration is enabled, this is the right time * to update the status of the page. */ if (kbase_is_page_migration_enabled() && !ignore_page_migration && - !is_huge(phys[i]) && !is_partial(phys[i])) - kbase_mmu_progress_migration_on_insert(phys[i], reg, mmut, - insert_vpfn + i); + !is_huge(base_tagged_addr) && !is_partial(base_tagged_addr)) + kbase_mmu_progress_migration_on_insert( + base_tagged_addr, reg, mmut, insert_vpfn + i); } num_of_valid_entries += count; } @@ -2491,7 +2448,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm } } - phys += count; + phys += (count / GPU_PAGES_PER_CPU_PAGE); insert_vpfn += count; remain -= count; kbase_kunmap(p, pgd_page); @@ -2507,10 +2464,11 @@ fail_unlock_free_pgds: kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]); fail_unlock: - if (insert_vpfn != start_vpfn) { + if (insert_vpfn != (start_vpfn * GPU_PAGES_PER_CPU_PAGE)) { /* Invalidate the pages we have partially completed */ - mmu_insert_pages_failure_recovery(kbdev, mmut, start_vpfn, insert_vpfn, dirty_pgds, - start_phys, ignore_page_migration); + mmu_insert_pages_failure_recovery(kbdev, mmut, start_vpfn * GPU_PAGES_PER_CPU_PAGE, + insert_vpfn, dirty_pgds, start_phys, + ignore_page_migration); } mmu_flush_invalidate_insert_pages(kbdev, mmut, start_vpfn, nr, @@ -2638,7 +2596,9 @@ KBASE_EXPORT_TEST_API(kbase_mmu_update); void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) { lockdep_assert_held(&kbdev->hwaccess_lock); +#if !MALI_USE_CSF lockdep_assert_held(&kbdev->mmu_hw_mutex); +#endif kbdev->mmu_mode->disable_as(kbdev, as_nr); } @@ -2661,10 +2621,9 @@ void kbase_mmu_disable(struct kbase_context *kctx) KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); op_param.vpfn = 0; - op_param.nr = ~0; + op_param.nr = ~0U; op_param.op = KBASE_MMU_OP_FLUSH_MEM; op_param.kctx_id = kctx->id; op_param.mmu_sync_info = mmu_sync_info; @@ -2705,6 +2664,8 @@ void kbase_mmu_disable(struct kbase_context *kctx) kctx->tgid, kctx->id); } #else + lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); + CSTD_UNUSED(lock_err); /* @@ -2714,7 +2675,7 @@ void kbase_mmu_disable(struct kbase_context *kctx) * The job scheduler code will already be holding the locks and context * so just do the flush. */ - flush_err = kbase_mmu_hw_do_flush_locked(kbdev, &kbdev->as[kctx->as_nr], &op_param); + flush_err = kbase_mmu_hw_do_flush(kbdev, &kbdev->as[kctx->as_nr], &op_param); if (flush_err) { dev_err(kbdev->dev, "Flush for GPU page table update did not complete to disable AS %d for ctx %d_%d", @@ -2751,7 +2712,7 @@ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, u64 *current_page = kbase_kmap(p); unsigned int current_valid_entries = kbdev->mmu_mode->get_num_valid_entries(current_page); - int index = (vpfn >> ((3 - current_level) * 9)) & 0x1FF; + unsigned int index = (vpfn >> ((3 - current_level) * 9)) & 0x1FFU; /* We need to track every level that needs updating */ if (dirty_pgds) @@ -2856,6 +2817,9 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase lockdep_assert_held(&mmut->mmu_lock); kbase_mmu_reset_free_pgds_list(mmut); + /* Convert to GPU_PAGE_SIZE units. */ + vpfn *= GPU_PAGES_PER_CPU_PAGE; + nr *= GPU_PAGES_PER_CPU_PAGE; while (nr) { unsigned int index = vpfn & 0x1FF; @@ -2871,7 +2835,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase if (count > nr) count = nr; - /* need to check if this is a 2MB page or a 4kB */ + /* need to check if this is a 2MB page or a small page */ for (level = MIDGARD_MMU_TOPLEVEL; level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { phys_addr_t next_pgd; @@ -2989,12 +2953,12 @@ out: * * @kbdev: Pointer to kbase device. * @mmut: Pointer to GPU MMU page table. - * @vpfn: Start page frame number of the GPU virtual pages to unmap. + * @vpfn: Start page frame number (in PAGE_SIZE units) of the GPU virtual pages to unmap. * @phys: Array of physical pages currently mapped to the virtual * pages to unmap, or NULL. This is used for GPU cache maintenance * and page migration support. - * @nr_phys_pages: Number of physical pages to flush. - * @nr_virt_pages: Number of virtual pages whose PTEs should be destroyed. + * @nr_phys_pages: Number of physical pages (in PAGE_SIZE units) to flush. + * @nr_virt_pages: Number of virtual pages (in PAGE_SIZE units) whose PTEs should be destroyed. * @as_nr: Address space number, for GPU cache maintenance operations * that happen outside a specific kbase context. * @ignore_page_migration: Whether page migration metadata should be ignored. @@ -3120,12 +3084,12 @@ int kbase_mmu_teardown_imported_pages(struct kbase_device *kbdev, struct kbase_m * * @kbdev: Pointer to kbase device. * @mmut: The involved MMU table - * @vpfn: Virtual PFN (Page Frame Number) of the first page to update + * @vpfn: Virtual PFN (Page Frame Number), in PAGE_SIZE units, of the first page to update * @phys: Pointer to the array of tagged physical addresses of the physical * pages that are pointed to by the page table entries (that need to * be updated). The pointer should be within the reg->gpu_alloc->pages * array. - * @nr: Number of pages to update + * @nr: Number of pages (in PAGE_SIZE units) to update * @flags: Flags * @group_id: The physical memory group in which the page was allocated. * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). @@ -3155,6 +3119,9 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb if (nr == 0) return 0; + /* Convert to GPU_PAGE_SIZE units. */ + vpfn *= GPU_PAGES_PER_CPU_PAGE; + nr *= GPU_PAGES_PER_CPU_PAGE; mutex_lock(&mmut->mmu_lock); while (nr) { @@ -3168,7 +3135,8 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb if (count > nr) count = nr; - if (is_huge(*phys) && (index == index_in_large_page(*phys))) + if (is_huge(*phys) && + (index == (index_in_large_page(*phys) * GPU_PAGES_PER_CPU_PAGE))) cur_level = MIDGARD_MMU_LEVEL(2); err = mmu_get_pgd_at_level(kbdev, mmut, vpfn, cur_level, &pgd); @@ -3186,7 +3154,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb num_of_valid_entries = kbdev->mmu_mode->get_num_valid_entries(pgd_page); if (cur_level == MIDGARD_MMU_LEVEL(2)) { - int level_index = (vpfn >> 9) & 0x1FF; + unsigned int level_index = (vpfn >> 9) & 0x1FFU; struct tagged_addr *target_phys = phys - index_in_large_page(*phys); #ifdef CONFIG_MALI_BIFROST_DEBUG @@ -3199,13 +3167,22 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb kbase_dma_addr(p) + (level_index * sizeof(u64)), sizeof(u64), KBASE_MMU_OP_NONE); } else { - for (i = 0; i < count; i++) { + for (i = 0; i < count; i += GPU_PAGES_PER_CPU_PAGE) { + phys_addr_t base_phys_address = + as_phys_addr_t(phys[i / GPU_PAGES_PER_CPU_PAGE]); + unsigned int j; + + for (j = 0; j < GPU_PAGES_PER_CPU_PAGE; j++) { + phys_addr_t page_address = + base_phys_address + (j * GPU_PAGE_SIZE); #ifdef CONFIG_MALI_BIFROST_DEBUG - WARN_ON_ONCE(!kbdev->mmu_mode->ate_is_valid( - pgd_page[index + i], MIDGARD_MMU_BOTTOMLEVEL)); + WARN_ON_ONCE(!kbdev->mmu_mode->ate_is_valid( + pgd_page[index + i + j], MIDGARD_MMU_BOTTOMLEVEL)); #endif - pgd_page[index + i] = kbase_mmu_create_ate( - kbdev, phys[i], flags, MIDGARD_MMU_BOTTOMLEVEL, group_id); + pgd_page[index + i + j] = kbase_mmu_create_ate( + kbdev, as_tagged(page_address), flags, + MIDGARD_MMU_BOTTOMLEVEL, group_id); + } } /* MMU cache flush strategy is NONE because GPU cache maintenance @@ -3221,7 +3198,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb if (dirty_pgds && count > 0) *dirty_pgds |= 1ULL << cur_level; - phys += count; + phys += (count / GPU_PAGES_PER_CPU_PAGE); vpfn += count; nr -= count; @@ -3333,10 +3310,12 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p struct kbase_device *kbdev; phys_addr_t pgd; u64 *old_page, *new_page, *pgd_page, *target, vpfn; - int index, check_state, ret = 0; + unsigned int index; + int check_state, ret = 0; unsigned long hwaccess_flags = 0; unsigned int num_of_valid_entries; u8 vmap_count = 0; + u8 pgd_entries_to_sync = (level == MIDGARD_MMU_BOTTOMLEVEL) ? GPU_PAGES_PER_CPU_PAGE : 1; /* If page migration support is not compiled in, return with fault */ if (!IS_ENABLED(CONFIG_PAGE_MIGRATION_SUPPORT)) @@ -3355,7 +3334,7 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p vpfn = PGD_VPFN_LEVEL_GET_VPFN(page_md->data.pt_mapped.pgd_vpfn_level); kbdev = mmut->kctx->kbdev; - index = (vpfn >> ((3 - level) * 9)) & 0x1FF; + index = (vpfn >> ((3 - level) * 9)) & 0x1FFU; /* Create all mappings before copying content. * This is done as early as possible because it is the only operation that may @@ -3405,8 +3384,8 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p #define PGD_VPFN_MASK(level) (~((((u64)1) << ((3 - level) * 9)) - 1)) op_param.mmu_sync_info = CALLER_MMU_ASYNC; op_param.kctx_id = mmut->kctx->id; - op_param.vpfn = vpfn & PGD_VPFN_MASK(level); - op_param.nr = 1 << ((3 - level) * 9); + op_param.vpfn = (vpfn / GPU_PAGES_PER_CPU_PAGE) & PGD_VPFN_MASK(level); + op_param.nr = 1U << ((3 - level) * 9); op_param.op = KBASE_MMU_OP_FLUSH_PT; /* When level is not MIDGARD_MMU_BOTTOMLEVEL, it is assumed PGD page migration */ op_param.flush_skip_levels = (level == MIDGARD_MMU_BOTTOMLEVEL) ? @@ -3525,7 +3504,7 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p /* Remap GPU virtual page. * * This code rests on the assumption that page migration is only enabled - * for 4 kB pages, that necessarily live in the bottom level of the MMU + * for small pages, that necessarily live in the bottom level of the MMU * page table. For this reason, the PGD level tells us inequivocably * whether the page being migrated is a "content page" or another PGD * of the page table: @@ -3545,10 +3524,18 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p num_of_valid_entries = kbdev->mmu_mode->get_num_valid_entries(pgd_page); if (level == MIDGARD_MMU_BOTTOMLEVEL) { - WARN_ON_ONCE((*target & 1UL) == 0); - *target = kbase_mmu_create_ate(kbdev, new_phys, page_md->data.mapped.reg->flags, - level, - page_md->data.mapped.reg->gpu_alloc->group_id); + phys_addr_t base_phys_address = as_phys_addr_t(new_phys); + unsigned int i; + + for (i = 0; i < GPU_PAGES_PER_CPU_PAGE; i++) { + phys_addr_t page_address = base_phys_address + (i * GPU_PAGE_SIZE); + + WARN_ON_ONCE((*target & 1UL) == 0); + *target = kbase_mmu_create_ate( + kbdev, as_tagged(page_address), page_md->data.mapped.reg->flags, + level, page_md->data.mapped.reg->gpu_alloc->group_id); + target++; + } } else { u64 managed_pte; @@ -3566,12 +3553,14 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p kbdev->mmu_mode->set_num_valid_entries(pgd_page, num_of_valid_entries); - /* This function always updates a single entry inside an existing PGD, - * therefore cache maintenance is necessary and affects a single entry. + /* This function always updates a single entry inside an existing PGD when + * level != MIDGARD_MMU_BOTTOMLEVEL, and would update more than one entry for + * MIDGARD_MMU_BOTTOMLEVEL PGD when PAGE_SIZE is not 4K, therefore cache + * maintenance is necessary. */ kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (index * sizeof(u64)), - kbase_dma_addr(phys_to_page(pgd)) + (index * sizeof(u64)), sizeof(u64), - KBASE_MMU_OP_FLUSH_PT); + kbase_dma_addr(phys_to_page(pgd)) + (index * sizeof(u64)), + pgd_entries_to_sync * sizeof(u64), KBASE_MMU_OP_FLUSH_PT); /* Unlock MMU region. * @@ -3612,8 +3601,8 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p /* Undertaking metadata transfer, while we are holding the mmu_lock */ spin_lock(&page_md->migrate_lock); if (level == MIDGARD_MMU_BOTTOMLEVEL) { - size_t page_array_index = - page_md->data.mapped.vpfn - page_md->data.mapped.reg->start_pfn; + size_t page_array_index = (page_md->data.mapped.vpfn / GPU_PAGES_PER_CPU_PAGE) - + page_md->data.mapped.reg->start_pfn; WARN_ON(PAGE_STATUS_GET(page_md->status) != ALLOCATED_MAPPED); @@ -3652,7 +3641,7 @@ undo_mappings: } static void mmu_teardown_level(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, - phys_addr_t pgd, unsigned int level) + phys_addr_t pgd, int level) { u64 *pgd_page; int i; @@ -3829,7 +3818,7 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, /* A modified physical address that contains * the page table level */ - u64 m_pgd = pgd | level; + u64 m_pgd = pgd | (u64)level; /* Put the modified physical address in the output buffer */ memcpy(*buffer, &m_pgd, sizeof(m_pgd)); @@ -3866,7 +3855,7 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, return size; } -void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) +void *kbase_mmu_dump(struct kbase_context *kctx, size_t nr_pages) { void *kaddr; size_t size_left; @@ -3942,7 +3931,7 @@ KBASE_EXPORT_TEST_API(kbase_mmu_dump); void kbase_mmu_bus_fault_worker(struct work_struct *data) { struct kbase_as *faulting_as; - int as_no; + unsigned int as_no; struct kbase_context *kctx; struct kbase_device *kbdev; struct kbase_fault *fault; diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h index 73f41b4d94cf..03bc066f897f 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.h @@ -23,12 +23,15 @@ #define _KBASE_MMU_H_ #include +#include #define KBASE_MMU_PAGE_ENTRIES 512 struct kbase_context; +struct kbase_device; struct kbase_mmu_table; struct kbase_va_region; +struct tagged_addr; /** * enum kbase_caller_mmu_sync_info - MMU-synchronous caller info. @@ -165,9 +168,9 @@ int kbase_mmu_insert_pages(struct kbase_device *kbdev, struct kbase_mmu_table *m * * @kbdev: Instance of GPU platform device, allocated from the probe method. * @mmut: GPU page tables. - * @vpfn: Start page frame number of the GPU virtual pages to map. + * @vpfn: Start page frame number (in PAGE_SIZE units) of the GPU virtual pages to map. * @phys: Physical address of the page to be mapped. - * @nr: The number of pages to map. + * @nr: The number of pages (in PAGE_SIZE units) to map. * @flags: Bitmask of attributes of the GPU memory region being mapped. * @as_nr: The GPU address space number. * @group_id: The physical memory group in which the page was allocated. @@ -217,11 +220,11 @@ int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, struct tagged_a * kbase_mmu_update_csf_mcu_pages - Update MCU mappings with changes of phys and flags * * @kbdev: Pointer to kbase device. - * @vpfn: Virtual PFN (Page Frame Number) of the first page to update + * @vpfn: GPU Virtual PFN (Page Frame Number), in PAGE_SIZE units, of the first page to update * @phys: Pointer to the array of tagged physical addresses of the physical * pages that are pointed to by the page table entries (that need to * be updated). - * @nr: Number of pages to update + * @nr: Number of pages (in PAGE_SIZE units) to update * @flags: Flags * @group_id: The physical memory group in which the page was allocated. * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h index b0b1837ae18b..c2b377de54a9 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h @@ -56,8 +56,8 @@ enum kbase_mmu_fault_type { /** * struct kbase_mmu_hw_op_param - parameters for kbase_mmu_hw_do_* functions - * @vpfn: MMU Virtual Page Frame Number to start the operation on. - * @nr: Number of pages to work on. + * @vpfn: MMU Virtual Page Frame Number (in PAGE_SIZE units) to start the operation on. + * @nr: Number of pages (in PAGE_SIZE units) to work on. * @op: Operation type (written to AS_COMMAND). * @kctx_id: Kernel context ID for MMU command tracepoint. * @mmu_sync_info: Indicates whether this call is synchronous wrt MMU ops. @@ -141,32 +141,13 @@ int kbase_mmu_hw_do_unlock(struct kbase_device *kbdev, struct kbase_as *as, * Issue a flush operation on the address space as per the information * specified inside @op_param. This function should not be called for * GPUs where MMU command to flush the cache(s) is deprecated. - * mmu_hw_mutex needs to be held when calling this function. + * hwaccess_lock needs to be held when calling this function. * * Return: 0 if the operation was successful, non-zero otherwise. */ int kbase_mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, const struct kbase_mmu_hw_op_param *op_param); -/** - * kbase_mmu_hw_do_flush_locked - Issue a flush operation to the MMU. - * - * @kbdev: Kbase device to issue the MMU operation on. - * @as: Address space to issue the MMU operation on. - * @op_param: Pointer to struct containing information about the MMU - * operation to perform. - * - * Issue a flush operation on the address space as per the information - * specified inside @op_param. This function should not be called for - * GPUs where MMU command to flush the cache(s) is deprecated. - * Both mmu_hw_mutex and hwaccess_lock need to be held when calling this - * function. - * - * Return: 0 if the operation was successful, non-zero otherwise. - */ -int kbase_mmu_hw_do_flush_locked(struct kbase_device *kbdev, struct kbase_as *as, - const struct kbase_mmu_hw_op_param *op_param); - /** * kbase_mmu_hw_do_flush_on_gpu_ctrl - Issue a flush operation to the MMU. * diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c index a2f55b847f71..ba67ae0e01e9 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c @@ -123,7 +123,7 @@ static int lock_region(struct kbase_gpu_props const *gpu_props, u64 *lockaddr, * therefore the highest bit that differs is bit #16 * and the region size (as a logarithm) is 16 + 1 = 17, i.e. 128 kB. */ - lockaddr_size_log2 = fls64(lockaddr_base ^ lockaddr_end); + lockaddr_size_log2 = (u64)fls64(lockaddr_base ^ lockaddr_end); /* Cap the size against minimum and maximum values allowed. */ if (lockaddr_size_log2 > KBASE_LOCK_REGION_MAX_SIZE_LOG2) @@ -166,25 +166,18 @@ static int lock_region(struct kbase_gpu_props const *gpu_props, u64 *lockaddr, */ static int wait_ready(struct kbase_device *kbdev, unsigned int as_nr) { - const ktime_t wait_loop_start = ktime_get_raw(); - const u32 mmu_as_inactive_wait_time_ms = kbdev->mmu_or_gpu_cache_op_wait_time_ms; - s64 diff; + u32 val; + int err; + const u32 timeout_us = + kbase_get_timeout_ms(kbdev, MMU_AS_INACTIVE_WAIT_TIMEOUT) * USEC_PER_MSEC; if (unlikely(kbdev->mmu_unresponsive)) return -EBUSY; - do { - unsigned int i; - - for (i = 0; i < 1000; i++) { - /* Wait for the MMU status to indicate there is no active command */ - if (!(kbase_reg_read32(kbdev, MMU_AS_OFFSET(as_nr, STATUS)) & - AS_STATUS_AS_ACTIVE_EXT_MASK)) - return 0; - } - - diff = ktime_to_ms(ktime_sub(ktime_get_raw(), wait_loop_start)); - } while (diff < mmu_as_inactive_wait_time_ms); + err = kbase_reg_poll32_timeout(kbdev, MMU_AS_OFFSET(as_nr, STATUS), val, + !(val & AS_STATUS_AS_ACTIVE_EXT_MASK), 0, timeout_us, false); + if (!err) + return 0; dev_err(kbdev->dev, "AS_ACTIVE bit stuck for as %u. Might be caused by unstable GPU clk/pwr or faulty system", @@ -196,14 +189,14 @@ static int wait_ready(struct kbase_device *kbdev, unsigned int as_nr) return -ETIMEDOUT; } -static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd) +static int write_cmd(struct kbase_device *kbdev, unsigned int as_nr, u32 cmd) { /* write AS_COMMAND when MMU is ready to accept another command */ const int status = wait_ready(kbdev, as_nr); if (likely(status == 0)) kbase_reg_write32(kbdev, MMU_AS_OFFSET(as_nr, COMMAND), - AS_COMMAND_COMMAND_SET(0, cmd)); + AS_COMMAND_COMMAND_SET(0U, cmd)); else if (status == -EBUSY) { dev_dbg(kbdev->dev, "Skipped the wait for AS_ACTIVE bit for as %u, before sending MMU command %u", @@ -217,7 +210,28 @@ static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd) return status; } -#if MALI_USE_CSF && !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) +#if MALI_USE_CSF +static int wait_l2_power_trans_complete(struct kbase_device *kbdev) +{ + u32 val; + const u32 timeout_us = + kbase_get_timeout_ms(kbdev, MMU_AS_INACTIVE_WAIT_TIMEOUT) * USEC_PER_MSEC; + const int err = kbase_reg_poll64_timeout(kbdev, GPU_CONTROL_ENUM(L2_PWRTRANS), val, + val == 0, 1, timeout_us, false); + + lockdep_assert_held(&kbdev->hwaccess_lock); + + if (err) { + dev_warn(kbdev->dev, "L2_PWRTRANS %016llx set for too long", + kbase_reg_read64(kbdev, GPU_CONTROL_ENUM(L2_PWRTRANS))); + if (kbase_prepare_to_reset_gpu_locked(kbdev, RESET_FLAGS_NONE)) + kbase_reset_gpu_locked(kbdev); + } + + return err; +} + +#if !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) static int wait_cores_power_trans_complete(struct kbase_device *kbdev) { #define WAIT_TIMEOUT 50000 /* 50ms timeout */ @@ -301,7 +315,8 @@ static int apply_hw_issue_GPU2019_3901_wa(struct kbase_device *kbdev, u32 *mmu_c return ret; } -#endif +#endif /* !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) */ +#endif /* MALI_USE_CSF */ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) { @@ -309,7 +324,9 @@ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) u64 transcfg = 0; lockdep_assert_held(&kbdev->hwaccess_lock); +#if !MALI_USE_CSF lockdep_assert_held(&kbdev->mmu_hw_mutex); +#endif transcfg = current_setup->transcfg; @@ -364,7 +381,7 @@ static void mmu_command_instr(struct kbase_device *kbdev, u32 kctx_id, u32 cmd, /* Helper function to program the LOCKADDR register before LOCK/UNLOCK command * is issued. */ -static int mmu_hw_set_lock_addr(struct kbase_device *kbdev, int as_nr, u64 *lock_addr, +static int mmu_hw_set_lock_addr(struct kbase_device *kbdev, unsigned int as_nr, u64 *lock_addr, const struct kbase_mmu_hw_op_param *op_param) { int ret; @@ -482,22 +499,15 @@ int kbase_mmu_hw_do_unlock(struct kbase_device *kbdev, struct kbase_as *as, return ret; } -/** - * mmu_hw_do_flush - Flush MMU and wait for its completion. - * - * @kbdev: Kbase device to issue the MMU operation on. - * @as: Address space to issue the MMU operation on. - * @op_param: Pointer to a struct containing information about the MMU operation. - * @hwaccess_locked: Flag to indicate if the lock has been held. - * - * Return: 0 if flushing MMU was successful, otherwise an error code. - */ -static int mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, - const struct kbase_mmu_hw_op_param *op_param, bool hwaccess_locked) +int kbase_mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, + const struct kbase_mmu_hw_op_param *op_param) { int ret; u64 lock_addr = 0x0; u32 mmu_cmd = AS_COMMAND_COMMAND_FLUSH_MEM; + const enum kbase_mmu_op_type flush_op = op_param->op; + + lockdep_assert_held(&kbdev->hwaccess_lock); if (WARN_ON(kbdev == NULL) || WARN_ON(as == NULL)) return -EINVAL; @@ -505,14 +515,12 @@ static int mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, /* MMU operations can be either FLUSH_PT or FLUSH_MEM, anything else at * this point would be unexpected. */ - if (op_param->op != KBASE_MMU_OP_FLUSH_PT && op_param->op != KBASE_MMU_OP_FLUSH_MEM) { + if (flush_op != KBASE_MMU_OP_FLUSH_PT && flush_op != KBASE_MMU_OP_FLUSH_MEM) { dev_err(kbdev->dev, "Unexpected flush operation received"); return -EINVAL; } - lockdep_assert_held(&kbdev->mmu_hw_mutex); - - if (op_param->op == KBASE_MMU_OP_FLUSH_PT) + if (flush_op == KBASE_MMU_OP_FLUSH_PT) mmu_cmd = AS_COMMAND_COMMAND_FLUSH_PT; /* Lock the region that needs to be updated */ @@ -524,16 +532,7 @@ static int mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, /* WA for the BASE_HW_ISSUE_GPU2019_3901. */ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2019_3901) && mmu_cmd == AS_COMMAND_COMMAND_FLUSH_MEM) { - if (!hwaccess_locked) { - unsigned long flags = 0; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - ret = apply_hw_issue_GPU2019_3901_wa(kbdev, &mmu_cmd, as->number); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } else { - ret = apply_hw_issue_GPU2019_3901_wa(kbdev, &mmu_cmd, as->number); - } - + ret = apply_hw_issue_GPU2019_3901_wa(kbdev, &mmu_cmd, as->number); if (ret) { dev_warn( kbdev->dev, @@ -543,8 +542,6 @@ static int mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, /* Continue with the MMU flush operation */ } } -#else - CSTD_UNUSED(hwaccess_locked); #endif ret = write_cmd(kbdev, as->number, mmu_cmd); @@ -553,32 +550,26 @@ static int mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, if (likely(!ret)) ret = wait_ready(kbdev, as->number); - if (likely(!ret)) + if (likely(!ret)) { mmu_command_instr(kbdev, op_param->kctx_id, mmu_cmd, lock_addr, op_param->mmu_sync_info); +#if MALI_USE_CSF + if (flush_op == KBASE_MMU_OP_FLUSH_MEM && + kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa && + kbdev->pm.backend.l2_state == KBASE_L2_PEND_OFF) + ret = wait_l2_power_trans_complete(kbdev); +#endif + } return ret; } -int kbase_mmu_hw_do_flush_locked(struct kbase_device *kbdev, struct kbase_as *as, - const struct kbase_mmu_hw_op_param *op_param) -{ - lockdep_assert_held(&kbdev->hwaccess_lock); - - return mmu_hw_do_flush(kbdev, as, op_param, true); -} - -int kbase_mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, - const struct kbase_mmu_hw_op_param *op_param) -{ - return mmu_hw_do_flush(kbdev, as, op_param, false); -} - int kbase_mmu_hw_do_flush_on_gpu_ctrl(struct kbase_device *kbdev, struct kbase_as *as, const struct kbase_mmu_hw_op_param *op_param) { int ret, ret2; u32 gpu_cmd = GPU_COMMAND_CACHE_CLN_INV_L2_LSC; + const enum kbase_mmu_op_type flush_op = op_param->op; if (WARN_ON(kbdev == NULL) || WARN_ON(as == NULL)) return -EINVAL; @@ -586,15 +577,14 @@ int kbase_mmu_hw_do_flush_on_gpu_ctrl(struct kbase_device *kbdev, struct kbase_a /* MMU operations can be either FLUSH_PT or FLUSH_MEM, anything else at * this point would be unexpected. */ - if (op_param->op != KBASE_MMU_OP_FLUSH_PT && op_param->op != KBASE_MMU_OP_FLUSH_MEM) { + if (flush_op != KBASE_MMU_OP_FLUSH_PT && flush_op != KBASE_MMU_OP_FLUSH_MEM) { dev_err(kbdev->dev, "Unexpected flush operation received"); return -EINVAL; } lockdep_assert_held(&kbdev->hwaccess_lock); - lockdep_assert_held(&kbdev->mmu_hw_mutex); - if (op_param->op == KBASE_MMU_OP_FLUSH_PT) + if (flush_op == KBASE_MMU_OP_FLUSH_PT) gpu_cmd = GPU_COMMAND_CACHE_CLN_INV_L2; /* 1. Issue MMU_AS_CONTROL.COMMAND.LOCK operation. */ @@ -608,6 +598,15 @@ int kbase_mmu_hw_do_flush_on_gpu_ctrl(struct kbase_device *kbdev, struct kbase_a /* 3. Issue MMU_AS_CONTROL.COMMAND.UNLOCK operation. */ ret2 = kbase_mmu_hw_do_unlock_no_addr(kbdev, as, op_param); +#if MALI_USE_CSF + if (!ret && !ret2) { + if (flush_op == KBASE_MMU_OP_FLUSH_MEM && + kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa && + kbdev->pm.backend.l2_state == KBASE_L2_PEND_OFF) + ret = wait_l2_power_trans_complete(kbdev); + } +#endif + return ret ?: ret2; } diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c index 3e0fab6e64f8..d19579da2f5d 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c @@ -74,7 +74,7 @@ static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) struct kbase_mmu_setup *const current_setup = &as->current_setup; current_setup->transtab = 0ULL; - current_setup->transcfg = AS_TRANSCFG_MODE_SET(0, AS_TRANSCFG_MODE_UNMAPPED); + current_setup->transcfg = AS_TRANSCFG_MODE_SET(0ULL, AS_TRANSCFG_MODE_UNMAPPED); /* Apply the address space setting */ kbase_mmu_hw_configure(kbdev, as); @@ -86,7 +86,7 @@ static phys_addr_t pte_to_phy_addr(u64 entry) return 0; entry &= ~VALID_ENTRY_MASK; - return entry & ~0xFFF; + return entry & ~0xFFFULL; } static int ate_is_valid(u64 ate, int const level) diff --git a/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_runtime_pm.c b/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_runtime_pm.c index 45f7638ad904..bd3b4b5e2aa3 100644 --- a/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_runtime_pm.c +++ b/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_runtime_pm.c @@ -49,7 +49,7 @@ static int resets_init(struct kbase_device *kbdev) return nr_resets; } - resets = devm_kcalloc(kbdev->dev, nr_resets, sizeof(*resets), GFP_KERNEL); + resets = devm_kcalloc(kbdev->dev, (size_t)nr_resets, sizeof(*resets), GFP_KERNEL); if (!resets) return -ENOMEM; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_kprobe.h similarity index 67% rename from drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h rename to drivers/gpu/arm/bifrost/tests/include/kutf/kutf_kprobe.h index f0368c2e59b9..f75cd776c60e 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_backend_config.h +++ b/drivers/gpu/arm/bifrost/tests/include/kutf/kutf_kprobe.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -19,11 +19,15 @@ * */ -/* - * Backend specific configuration - */ +#ifndef _KUTF_KPROBE_H_ +#define _KUTF_KPROBE_H_ -#ifndef _KBASE_BACKEND_CONFIG_H_ -#define _KBASE_BACKEND_CONFIG_H_ +int kutf_kprobe_init(struct dentry *base_dir); +void kutf_kprobe_exit(void); -#endif /* _KBASE_BACKEND_CONFIG_H_ */ +typedef void (*kutf_kp_handler)(int argc, char **argv); + +void kutf_kp_sample_handler(int argc, char **argv); +void kutf_kp_sample_kernel_function(void); + +#endif /* _KUTF_KPROBE_H_ */ diff --git a/drivers/gpu/arm/bifrost/tests/kutf/Kbuild b/drivers/gpu/arm/bifrost/tests/kutf/Kbuild index c4790bc66c23..aef44e05d0ea 100644 --- a/drivers/gpu/arm/bifrost/tests/kutf/Kbuild +++ b/drivers/gpu/arm/bifrost/tests/kutf/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2017, 2020-2023 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -27,5 +27,6 @@ kutf-y := \ kutf_suite.o \ kutf_utils.o \ kutf_helpers.o \ - kutf_helpers_user.o + kutf_helpers_user.o \ + kutf_kprobe.o endif diff --git a/drivers/gpu/arm/bifrost/tests/kutf/build.bp b/drivers/gpu/arm/bifrost/tests/kutf/build.bp index 89edae9c5e6f..267c14bb3a8b 100644 --- a/drivers/gpu/arm/bifrost/tests/kutf/build.bp +++ b/drivers/gpu/arm/bifrost/tests/kutf/build.bp @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -34,6 +34,7 @@ bob_kernel_module { "kutf_resultset.c", "kutf_suite.c", "kutf_utils.c", + "kutf_kprobe.c", ], enabled: false, mali_kutf: { diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c index 8654fd503960..a69c791d97b0 100644 --- a/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_helpers_user.c @@ -56,7 +56,7 @@ static const char *get_val_type_name(enum kutf_helper_valtype valtype) * - Has between 1 and KUTF_HELPER_MAX_VAL_NAME_LEN characters before the \0 terminator * - And, each char is in the character set [A-Z0-9_] */ -static int validate_val_name(const char *val_str, int str_len) +static int validate_val_name(const char *val_str, size_t str_len) { int i = 0; @@ -92,16 +92,16 @@ static int validate_val_name(const char *val_str, int str_len) * That is, before any '\\', '\n' or '"' characters. This is so we don't have * to escape the string */ -static int find_quoted_string_valid_len(const char *str) +static size_t find_quoted_string_valid_len(const char *str) { char *ptr; const char *check_chars = "\\\n\""; ptr = strpbrk(str, check_chars); if (ptr) - return (int)(ptr - str); + return (size_t)(ptr - str); - return (int)strlen(str); + return strlen(str); } static int kutf_helper_userdata_enqueue(struct kutf_context *context, const char *str) @@ -116,7 +116,7 @@ static int kutf_helper_userdata_enqueue(struct kutf_context *context, const char if (!str_copy) return -ENOMEM; - strcpy(str_copy, str); + strscpy(str_copy, str, len); err = kutf_add_result(context, KUTF_RESULT_USERDATA, str_copy); @@ -185,14 +185,14 @@ EXPORT_SYMBOL(kutf_helper_max_str_len_for_kern); int kutf_helper_send_named_str(struct kutf_context *context, const char *val_name, const char *val_str) { - int val_str_len; - int str_buf_sz; + size_t val_str_len; + size_t str_buf_sz; char *str_buf = NULL; int ret = 1; char *copy_ptr; - int val_name_len; - int start_delim_len = strlen(NAMED_STR_START_DELIM); - int end_delim_len = strlen(NAMED_STR_END_DELIM); + size_t val_name_len; + size_t start_delim_len = strlen(NAMED_STR_START_DELIM); + size_t end_delim_len = strlen(NAMED_STR_END_DELIM); const char *errmsg = NULL; if (validate_val_name(val_name, KUTF_HELPER_MAX_VAL_NAME_LEN + 1)) { @@ -215,7 +215,7 @@ int kutf_helper_send_named_str(struct kutf_context *context, const char *val_nam if (!str_buf) { errmsg = kutf_dsprintf( &context->fixture_pool, - "Failed to send str value named '%s': kmalloc failed, str_buf_sz=%d", + "Failed to send str value named '%s': kmalloc failed, str_buf_sz=%zu", val_name, str_buf_sz); goto out_err; } @@ -270,8 +270,8 @@ int kutf_helper_receive_named_val(struct kutf_context *context, char *recv_str; char *search_ptr; char *name_str = NULL; - int name_len; - int strval_len; + size_t name_len; + size_t strval_len; enum kutf_helper_valtype type = KUTF_HELPER_VALTYPE_INVALID; char *strval = NULL; u64 u64val = 0; @@ -286,7 +286,7 @@ int kutf_helper_receive_named_val(struct kutf_context *context, /* Find the '=', grab the name and validate it */ search_ptr = strnchr(recv_str, recv_sz, NAMED_VALUE_SEP[0]); if (search_ptr) { - name_len = search_ptr - recv_str; + name_len = (size_t)(search_ptr - recv_str); if (!validate_val_name(recv_str, name_len)) { /* no need to reallocate - just modify string in place */ name_str = recv_str; @@ -311,7 +311,7 @@ int kutf_helper_receive_named_val(struct kutf_context *context, /* Find end of string */ search_ptr = strnchr(recv_str, recv_sz, NAMED_STR_END_DELIM[0]); if (search_ptr) { - strval_len = search_ptr - recv_str; + strval_len = (size_t)(search_ptr - recv_str); /* Validate the string to ensure it contains no quotes */ if (strval_len == find_quoted_string_valid_len(recv_str)) { /* no need to reallocate - just modify string in place */ @@ -339,7 +339,7 @@ int kutf_helper_receive_named_val(struct kutf_context *context, * reads characters after the number it'll report -EINVAL */ if (!err) { - int len_remain = strnlen(recv_str, recv_sz); + size_t len_remain = strnlen(recv_str, recv_sz); type = KUTF_HELPER_VALTYPE_U64; recv_str += len_remain; diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_kprobe.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_kprobe.c new file mode 100644 index 000000000000..f118692c43a1 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_kprobe.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +/* Kernel probe functionality. + */ + +#ifdef CONFIG_KPROBES + +#include +#include +#include +#include +#include +#include + +#define KUTF_KP_REG_MIN_ARGS 3 +#define KUTF_KP_UNREG_MIN_ARGS 2 +#define KUTF_KP_FUNC_NAME_ARG 0 +#define KUTF_KP_FUNC_ENTRY_EXIT_ARG 1 +#define KUTF_KP_FUNC_HANDLER_ARG 2 + +#define KUTF_KP_WRITE_BUFSIZE 4096 + +/* Stores address to kernel function 'kallsyms_lookup_name' + * as 'kallsyms_lookup_name' is no longer exported from 5.7 kernel. + */ +typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); +kallsyms_lookup_name_t kutf_ksym_lookup_name; + +static ssize_t kutf_kp_reg_debugfs_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos); + +static ssize_t kutf_kp_unreg_debugfs_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos); + +static LIST_HEAD(kp_list); +static DEFINE_MUTEX(kp_list_lock); + +/** + * struct kutf_kp_data - Structure which holds data per kprobe instance + * @kretp: reference to kernel ret probe + * @entry: true if this probe is for function entry.Otherwise it is exit. + * @argc: Number of arguments to be passed to probe handler + * @argv: arguments passed to probe handler + * @kp_handler: Actual handler which is called when probe is triggered + * @list: node for adding to kp_list + */ +struct kutf_kp_data { + struct kretprobe kretp; + bool entry; + int argc; + char **argv; + kutf_kp_handler kp_handler; + struct list_head list; +}; + +const struct file_operations kutf_kp_reg_debugfs_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = kutf_kp_reg_debugfs_write, +}; + +const struct file_operations kutf_kp_unreg_debugfs_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = kutf_kp_unreg_debugfs_write, +}; + +struct kprobe kutf_kallsym_kp = { .symbol_name = "kallsyms_lookup_name" }; + +void kutf_kp_sample_kernel_function(void) +{ + pr_debug("%s called\n", __func__); +} +EXPORT_SYMBOL(kutf_kp_sample_kernel_function); + +void kutf_kp_sample_handler(int argc, char **argv) +{ + int i = 0; + + for (; i < argc; i++) + pr_info("%s %s\n", __func__, argv[i]); +} +EXPORT_SYMBOL(kutf_kp_sample_handler); + +static int kutf_call_kp_handler(struct kretprobe *p) +{ + struct kutf_kp_data *kp_p; + kutf_kp_handler kp_handler; + + kp_p = (struct kutf_kp_data *)p; + kp_handler = kp_p->kp_handler; + + if (kp_handler) { + /* Arguments to registered handler starts after + * KUTF_KP_REG_MIN_ARGS. + */ + (*kp_handler)((kp_p->argc) - KUTF_KP_REG_MIN_ARGS, + &(kp_p->argv[KUTF_KP_REG_MIN_ARGS])); + } + return 0; +} + +static int kutf_kretp_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ +#if (KERNEL_VERSION(5, 11, 0) <= LINUX_VERSION_CODE) + return kutf_call_kp_handler(ri->rph->rp); +#else + return kutf_call_kp_handler(ri->rp); +#endif +} + +static kutf_kp_handler kutf_get_kp_handler(char **argv) +{ + if ((NULL == argv) || (NULL == kutf_ksym_lookup_name)) + return NULL; + + return (kutf_kp_handler)((*kutf_ksym_lookup_name)(argv[KUTF_KP_FUNC_HANDLER_ARG])); +} + +static ssize_t kutf_kp_reg_debugfs_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + int argc = 0; + int ret = count; + char **argv; + char *kbuf; + char *func_name; + struct kutf_kp_data *kp_p; + struct kutf_kp_data *kp_iter; + + if (count >= KUTF_KP_WRITE_BUFSIZE) + return -EINVAL; + + kbuf = memdup_user(user_buf, count); + if (IS_ERR(kbuf)) { + return -ENOMEM; + } + kbuf[count - 1] = '\0'; + + argv = argv_split(GFP_KERNEL, kbuf, &argc); + if (!argv) { + ret = -ENOMEM; + goto out; + } + if (argc < KUTF_KP_REG_MIN_ARGS) { + pr_debug("Insufficient args in reg kprobe:%d\n", argc); + ret = -EINVAL; + goto argv_out; + } + + kp_p = kzalloc(sizeof(struct kutf_kp_data), GFP_KERNEL); + if (!kp_p) + goto argv_out; + + if (!(strcmp(argv[KUTF_KP_FUNC_ENTRY_EXIT_ARG], "entry"))) { + kp_p->kretp.entry_handler = kutf_kretp_handler; + kp_p->entry = 1; + } else if (!(strcmp(argv[KUTF_KP_FUNC_ENTRY_EXIT_ARG], "exit"))) { + kp_p->kretp.handler = kutf_kretp_handler; + } else { + pr_err("Invalid arg:%s passed in reg kprobe\n", argv[KUTF_KP_FUNC_ENTRY_EXIT_ARG]); + ret = -EINVAL; + kfree(kp_p); + goto argv_out; + } + + func_name = argv[KUTF_KP_FUNC_NAME_ARG]; + + mutex_lock(&kp_list_lock); + list_for_each_entry(kp_iter, &kp_list, list) { + if ((kp_iter->entry == kp_p->entry) && + (!(strcmp(kp_iter->kretp.kp.symbol_name, func_name)))) { + ret = -EEXIST; + kfree(kp_p); + mutex_unlock(&kp_list_lock); + goto argv_out; + } + } + + kp_p->kretp.kp.symbol_name = func_name; + kp_p->kp_handler = kutf_get_kp_handler(argv); + if (!(kp_p->kp_handler)) { + pr_debug("cannot find addr for handler:%s\n", argv[KUTF_KP_FUNC_HANDLER_ARG]); + ret = -EINVAL; + kfree(kp_p); + mutex_unlock(&kp_list_lock); + goto argv_out; + } + kp_p->argc = argc; + kp_p->argv = argv; + ret = register_kretprobe(&kp_p->kretp); + if (ret) { + ret = -EINVAL; + kfree(kp_p); + mutex_unlock(&kp_list_lock); + goto argv_out; + } + INIT_LIST_HEAD(&kp_p->list); + list_add(&kp_p->list, &kp_list); + + mutex_unlock(&kp_list_lock); + + ret = count; + goto out; + +argv_out: + argv_free(argv); + +out: + kfree(kbuf); + + return ret; +} + +static ssize_t kutf_kp_unreg_debugfs_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + int argc = 0; + int ret = -EINVAL; + char **argv = NULL; + char *kbuf; + char *func_name; + struct kutf_kp_data *kp_iter; + bool entry; + + if (count >= KUTF_KP_WRITE_BUFSIZE) + return -EINVAL; + + kbuf = memdup_user(user_buf, count); + if (IS_ERR(kbuf)) { + return -ENOMEM; + } + kbuf[count - 1] = '\0'; + + argv = argv_split(GFP_KERNEL, kbuf, &argc); + if (!argv) { + ret = -ENOMEM; + goto out; + } + if (argc < KUTF_KP_UNREG_MIN_ARGS) { + pr_debug("Insufficient args in unreg kprobe:%d\n", argc); + ret = -EINVAL; + goto out; + } + if (!(strcmp(argv[KUTF_KP_FUNC_ENTRY_EXIT_ARG], "entry"))) + entry = 1; + else if (!(strcmp(argv[KUTF_KP_FUNC_ENTRY_EXIT_ARG], "exit"))) + entry = 0; + else { + pr_err("Invalid arg:%s passed in unreg kprobe\n", + argv[KUTF_KP_FUNC_ENTRY_EXIT_ARG]); + ret = -EINVAL; + goto out; + } + func_name = argv[KUTF_KP_FUNC_NAME_ARG]; + + mutex_lock(&kp_list_lock); + list_for_each_entry(kp_iter, &kp_list, list) { + if ((kp_iter->entry == entry) && + (!(strcmp(func_name, kp_iter->kretp.kp.symbol_name)))) { + unregister_kretprobe(&kp_iter->kretp); + argv_free(kp_iter->argv); + list_del(&kp_iter->list); + kfree(kp_iter); + ret = count; + break; + } + } + mutex_unlock(&kp_list_lock); +out: + argv_free(argv); + kfree(kbuf); + + return ret; +} + +int __init kutf_kprobe_init(struct dentry *base_dir) +{ + struct dentry *kutf_kp_reg_debugfs_file; + struct dentry *kutf_kp_unreg_debugfs_file; + + if (!(register_kprobe(&kutf_kallsym_kp))) { + /* After kernel 5.7, 'kallsyms_lookup_name' is no longer + * exported. So we need this workaround to get the + * addr of 'kallsyms_lookup_name'. This will be used later + * in kprobe handler function to call the registered + * handler for a probe from the name passed from userspace. + */ + kutf_ksym_lookup_name = (kallsyms_lookup_name_t)kutf_kallsym_kp.addr; + unregister_kprobe(&kutf_kallsym_kp); + kutf_kp_reg_debugfs_file = debugfs_create_file("register_kprobe", 0200, base_dir, + NULL, &kutf_kp_reg_debugfs_fops); + if (IS_ERR_OR_NULL(kutf_kp_reg_debugfs_file)) + pr_err("Failed to create kprobe reg debugfs file"); + + kutf_kp_unreg_debugfs_file = debugfs_create_file( + "unregister_kprobe", 0200, base_dir, NULL, &kutf_kp_unreg_debugfs_fops); + if (IS_ERR_OR_NULL(kutf_kp_unreg_debugfs_file)) { + pr_err("Failed to create kprobe unreg debugfs file"); + debugfs_remove(kutf_kp_reg_debugfs_file); + } + } else + pr_info("kallsyms_lookup_name addr not available\n"); + + return 0; +} + +void kutf_kprobe_exit(void) +{ + struct kutf_kp_data *kp_iter; + struct kutf_kp_data *n; + + mutex_lock(&kp_list_lock); + + list_for_each_entry_safe(kp_iter, n, &kp_list, list) { + unregister_kretprobe(&kp_iter->kretp); + argv_free(kp_iter->argv); + list_del(&kp_iter->list); + kfree(kp_iter); + } + + mutex_unlock(&kp_list_lock); +} + +#endif diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c index 9e57c10befdf..dcd4cb3f36fc 100644 --- a/drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_suite.c @@ -33,13 +33,15 @@ #include #include #include - #include #include #include #include #include +#ifdef CONFIG_KPROBES +#include +#endif /** * struct kutf_application - Structure which represents kutf application @@ -401,7 +403,7 @@ static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, size_t if (bytes_not_copied != 0) return -EFAULT; test_context->userdata.flags |= KUTF_USERDATA_WARNING_OUTPUT; - return message_len; + return (ssize_t)message_len; case KUTF_RESULT_USERDATA: message_len = strlen(res->message); if (message_len > len - 1) { @@ -419,7 +421,7 @@ static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, size_t pr_warn("Failed to copy data to user space buffer\n"); return -EFAULT; } - return message_len + 1; + return (ssize_t)message_len + 1; default: /* Fall through - this is a test result */ break; @@ -441,28 +443,28 @@ static ssize_t kutf_debugfs_run_read(struct file *file, char __user *buf, size_t /* First copy the result string */ if (kutf_str_ptr) { bytes_not_copied = copy_to_user(&buf[0], kutf_str_ptr, kutf_str_len); - bytes_copied += kutf_str_len - bytes_not_copied; + bytes_copied += (ssize_t)(kutf_str_len - bytes_not_copied); if (bytes_not_copied) goto exit; } /* Then the separator */ bytes_not_copied = copy_to_user(&buf[bytes_copied], &separator, 1); - bytes_copied += 1 - bytes_not_copied; + bytes_copied += (ssize_t)(1 - bytes_not_copied); if (bytes_not_copied) goto exit; /* Finally Next copy the result string */ if (res->message) { bytes_not_copied = copy_to_user(&buf[bytes_copied], res->message, message_len); - bytes_copied += message_len - bytes_not_copied; + bytes_copied += (ssize_t)(message_len - bytes_not_copied); if (bytes_not_copied) goto exit; } /* Finally the terminator */ bytes_not_copied = copy_to_user(&buf[bytes_copied], &terminator, 1); - bytes_copied += 1 - bytes_not_copied; + bytes_copied += (ssize_t)(1 - bytes_not_copied); exit: return bytes_copied; @@ -495,7 +497,7 @@ static ssize_t kutf_debugfs_run_write(struct file *file, const char __user *buf, if (ret < 0) return ret; - return len; + return (ssize_t)len; } /** @@ -1136,6 +1138,10 @@ static int __init init_kutf_core(void) return -ENOMEM; } +#ifdef CONFIG_KPROBES + kutf_kprobe_init(base_dir); +#endif + return 0; } @@ -1146,6 +1152,9 @@ static int __init init_kutf_core(void) */ static void __exit exit_kutf_core(void) { +#ifdef CONFIG_KPROBES + kutf_kprobe_exit(); +#endif debugfs_remove_recursive(base_dir); if (kutf_workq) diff --git a/drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c b/drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c index 6454a20b8ae9..183ab368ecbe 100644 --- a/drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c +++ b/drivers/gpu/arm/bifrost/tests/kutf/kutf_utils.c @@ -37,7 +37,7 @@ const char *kutf_dsprintf(struct kutf_mempool *pool, const char *fmt, ...) { va_list args; int len; - int size; + size_t size; void *buffer; mutex_lock(&buffer_lock); @@ -54,7 +54,7 @@ const char *kutf_dsprintf(struct kutf_mempool *pool, const char *fmt, ...) pr_warn("%s: Truncated dsprintf message %s\n", __func__, fmt); size = sizeof(tmp_buffer); } else { - size = len + 1; + size = (size_t)(len + 1); } buffer = kutf_mempool_alloc(pool, size); diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c index a221aa75a191..6c343cf9f73b 100644 --- a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c @@ -255,11 +255,11 @@ static const char *kutf_clk_trace_do_get_rate(struct kutf_context *context, spin_unlock(&kbdev->pm.clk_rtm.lock); if ((i + 1) == data->nclks) - ret += snprintf(portal_msg_buf + ret, PORTAL_MSG_LEN - ret, + ret += snprintf(portal_msg_buf + ret, PORTAL_MSG_LEN - (size_t)ret, "0x%lx], GPU_IDLE:%d}", rate, idle); else - ret += snprintf(portal_msg_buf + ret, PORTAL_MSG_LEN - ret, "0x%lx, ", - rate); + ret += snprintf(portal_msg_buf + ret, PORTAL_MSG_LEN - (size_t)ret, + "0x%lx, ", rate); if (ret >= PORTAL_MSG_LEN) { pr_warn("Message buf overflow with rate array data\n"); @@ -319,7 +319,7 @@ static const char *kutf_clk_trace_do_get_snapshot(struct kutf_context *context, fmt = "(0x%lx, 0x%lx, %u, %u)]}"; else fmt = "(0x%lx, 0x%lx, %u, %u), "; - ret += snprintf(portal_msg_buf + ret, PORTAL_MSG_LEN - ret, fmt, + ret += snprintf(portal_msg_buf + ret, PORTAL_MSG_LEN - (size_t)ret, fmt, snapshot.previous_rate, snapshot.current_rate, snapshot.rate_up_cnt, snapshot.rate_down_cnt); if (ret >= PORTAL_MSG_LEN) { diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c index 112aef433759..c0d46719bc71 100644 --- a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c @@ -51,56 +51,58 @@ struct kutf_irq_fixture_data { struct kbase_device *kbdev; }; -/* ID for the GPU IRQ */ -#define GPU_IRQ_HANDLER 2 +/* Tag for GPU IRQ */ +#define GPU_IRQ_TAG 2 #define NR_TEST_IRQS ((u32)1000000) -/* IRQ for the test to trigger. Currently POWER_CHANGED_SINGLE as it is - * otherwise unused in the DDK - */ -#define TEST_IRQ POWER_CHANGED_SINGLE - #define IRQ_TIMEOUT HZ -/* Kernel API for setting irq throttle hook callback and irq time in us*/ -extern int kbase_set_custom_irq_handler(struct kbase_device *kbdev, irq_handler_t custom_handler, - int irq_type); -extern irqreturn_t kbase_gpu_irq_test_handler(int irq, void *data, u32 val); +static void *kbase_untag(void *ptr) +{ + return (void *)(((uintptr_t)ptr) & ~(uintptr_t)3); +} static DECLARE_WAIT_QUEUE_HEAD(wait); static bool triggered; static u64 irq_time; -static void *kbase_untag(void *ptr) -{ - return (void *)(((uintptr_t)ptr) & ~3); -} - /** * kbase_gpu_irq_custom_handler - Custom IRQ throttle handler * @irq: IRQ number * @data: Data associated with this IRQ * - * Return: state of the IRQ + * Return: IRQ_HANDLED if any interrupt has been handled. IRQ_NONE otherwise. */ static irqreturn_t kbase_gpu_irq_custom_handler(int irq, void *data) { struct kbase_device *kbdev = kbase_untag(data); - u32 val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_STATUS)); + u32 status_reg_enum = GPU_CONTROL_ENUM(GPU_IRQ_STATUS); + u32 clear_reg_enum = GPU_CONTROL_ENUM(GPU_IRQ_CLEAR); + u32 test_irq = POWER_CHANGED_SINGLE; + u32 val = kbase_reg_read32(kbdev, status_reg_enum); irqreturn_t result; u64 tval; - bool has_test_irq = val & TEST_IRQ; + bool has_test_irq = val & test_irq; + if (has_test_irq) { tval = ktime_get_real_ns(); /* Clear the test source only here */ - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_CLEAR), TEST_IRQ); + kbase_reg_write32(kbdev, clear_reg_enum, test_irq); /* Remove the test IRQ status bit */ - val = val ^ TEST_IRQ; + val = val ^ test_irq; } - result = kbase_gpu_irq_test_handler(irq, data, val); + if (!val) + result = IRQ_NONE; + else { +#if IS_ENABLED(CONFIG_MALI_REAL_HW) + dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); + kbase_gpu_interrupt(kbdev, val); +#endif + result = IRQ_HANDLED; + } if (has_test_irq) { irq_time = tval; @@ -180,15 +182,17 @@ static void mali_kutf_irq_latency(struct kutf_context *context) kbase_pm_context_active(kbdev); kbase_pm_wait_for_desired_state(kbdev); - kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler, GPU_IRQ_HANDLER); + kbase_set_custom_irq_handler(kbdev, kbase_gpu_irq_custom_handler, GPU_IRQ_TAG); for (i = 1; i <= NR_TEST_IRQS; i++) { u64 start_time = ktime_get_real_ns(); + u32 reg_enum = GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT); + u32 test_irq = POWER_CHANGED_SINGLE; triggered = false; /* Trigger fake IRQ */ - kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(GPU_IRQ_RAWSTAT), TEST_IRQ); + kbase_reg_write32(kbdev, reg_enum, test_irq); if (wait_event_timeout(wait, triggered, IRQ_TIMEOUT) == 0) { /* Wait extra time to see if it would come */ @@ -211,7 +215,7 @@ static void mali_kutf_irq_latency(struct kutf_context *context) } /* Go back to default handler */ - kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_HANDLER); + kbase_set_custom_irq_handler(kbdev, NULL, GPU_IRQ_TAG); kbase_pm_context_idle(kbdev); diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c index 6b0c0ffbe6c4..8937d69f182f 100644 --- a/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c @@ -50,7 +50,7 @@ struct kutf_application *mgm_app; */ struct kutf_mgm_fixture_data { struct kbase_device *kbdev; - int group_id; + unsigned int group_id; }; /** diff --git a/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c b/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c index 1abcb8fd98a0..1592eab806ac 100644 --- a/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c +++ b/drivers/gpu/arm/bifrost/thirdparty/mali_kbase_mmap.c @@ -270,8 +270,8 @@ unsigned long kbase_context_get_unmapped_area(struct kbase_context *const kctx, unsigned long high_limit = mm->mmap_base; unsigned long low_limit = PAGE_SIZE; #endif - int cpu_va_bits = BITS_PER_LONG; - int gpu_pc_bits = kctx->kbdev->gpu_props.log2_program_counter_size; + unsigned int cpu_va_bits = BITS_PER_LONG; + unsigned int gpu_pc_bits = kctx->kbdev->gpu_props.log2_program_counter_size; bool is_shader_code = false; bool is_same_4gb_page = false; unsigned long ret; diff --git a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c index 77ebcb05e2b4..a91278dd3bef 100644 --- a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c +++ b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c @@ -42,7 +42,7 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) /* Summarize the Address Space objects. */ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) - __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], as_nr); + __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], (u32)as_nr); /* Create Legacy GPU object to track in AOM for dumping */ __kbase_tlstream_tl_new_gpu(summary, kbdev, kbdev->id, kbdev->gpu_props.num_cores); @@ -53,7 +53,7 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) /* Trace the creation of a new kbase device and set its properties. */ __kbase_tlstream_tl_kbase_new_device( summary, kbdev->id, kbdev->gpu_props.num_cores, kbdev->csf.global_iface.group_num, - kbdev->nr_hw_address_spaces, num_sb_entries, + (u32)kbdev->nr_hw_address_spaces, num_sb_entries, kbdev->gpu_props.gpu_features.cross_stream_sync, supports_gpu_sleep, 0 ); @@ -122,7 +122,7 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) /* Trace the currently assigned address space */ if (kctx->as_nr != KBASEP_AS_NR_INVALID) - __kbase_tlstream_tl_kbase_ctx_assign_as(body, kctx->id, kctx->as_nr); + __kbase_tlstream_tl_kbase_ctx_assign_as(body, kctx->id, (u32)kctx->as_nr); /* Trace all KCPU queues in the context into the body stream. * As we acquired the KCPU lock after resetting the body stream, diff --git a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c index 628f29aab9ff..3e9e6e864125 100644 --- a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c +++ b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_jm.c @@ -42,7 +42,7 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) /* Summarize the Address Space objects. */ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) - __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], as_nr); + __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], (u32)as_nr); /* Create GPU object and make it retain all LPUs and address spaces. */ __kbase_tlstream_tl_new_gpu(summary, kbdev, kbdev->id, kbdev->gpu_props.num_cores); diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c index ec38c70504b7..eaa5a848ede7 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline.c @@ -183,7 +183,7 @@ int kbase_timeline_acquire(struct kbase_device *kbdev, u32 flags) if (WARN_ON(!timeline)) return -EFAULT; - if (atomic_cmpxchg(timeline->timeline_flags, 0, timeline_flags)) + if (atomic_cmpxchg(timeline->timeline_flags, 0, (int)timeline_flags)) return -EBUSY; #if MALI_USE_CSF @@ -387,8 +387,8 @@ void kbase_timeline_stats(struct kbase_timeline *timeline, u32 *bytes_collected, /* Accumulate bytes generated per stream */ *bytes_generated = 0; for (stype = (enum tl_stream_type)0; stype < TL_STREAM_TYPE_COUNT; stype++) - *bytes_generated += atomic_read(&timeline->streams[stype].bytes_generated); + *bytes_generated += (u32)atomic_read(&timeline->streams[stype].bytes_generated); - *bytes_collected = atomic_read(&timeline->bytes_collected); + *bytes_collected = (u32)atomic_read(&timeline->bytes_collected); } #endif /* MALI_UNIT_TEST */ diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c index 5f3b79b6ecbd..d98e22880419 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c @@ -120,7 +120,7 @@ static int kbasep_timeline_io_packet_pending(struct kbase_timeline *timeline, for (i = (enum tl_stream_type)0; i < TL_STREAM_TYPE_COUNT; ++i) { struct kbase_tlstream *stream = &timeline->streams[i]; - *rb_idx_raw = atomic_read(&stream->rbi); + *rb_idx_raw = (unsigned int)atomic_read(&stream->rbi); /* Read buffer index may be updated by writer in case of * overflow. Read and write buffer indexes must be * loaded in correct order. @@ -169,7 +169,7 @@ static inline int copy_stream_header(char __user *buffer, size_t size, ssize_t * const char *hdr, size_t hdr_size, size_t *hdr_btc) { const size_t offset = hdr_size - *hdr_btc; - const size_t copy_size = MIN(size - *copy_len, *hdr_btc); + const size_t copy_size = MIN((size_t)((ssize_t)size - *copy_len), *hdr_btc); if (!*hdr_btc) return 0; @@ -181,7 +181,7 @@ static inline int copy_stream_header(char __user *buffer, size_t size, ssize_t * return -1; *hdr_btc -= copy_size; - *copy_len += copy_size; + *copy_len += (ssize_t)copy_size; return 0; } @@ -290,8 +290,8 @@ static ssize_t kbasep_timeline_io_read(struct file *filp, char __user *buffer, s * If so copy its content. */ rb_idx = rb_idx_raw % PACKET_COUNT; - rb_size = atomic_read(&stream->buffer[rb_idx].size); - if (rb_size > size - copy_len) + rb_size = (size_t)atomic_read(&stream->buffer[rb_idx].size); + if (rb_size > (size_t)((ssize_t)size - copy_len)) break; if (copy_to_user(&buffer[copy_len], stream->buffer[rb_idx].data, rb_size)) { copy_len = -EFAULT; @@ -304,10 +304,10 @@ static ssize_t kbasep_timeline_io_read(struct file *filp, char __user *buffer, s * that we have just sent to user. */ smp_rmb(); - wb_idx_raw = atomic_read(&stream->wbi); + wb_idx_raw = (unsigned int)atomic_read(&stream->wbi); if (wb_idx_raw - rb_idx_raw < PACKET_COUNT) { - copy_len += rb_size; + copy_len += (ssize_t)rb_size; atomic_inc(&stream->rbi); #if MALI_UNIT_TEST atomic_add(rb_size, &timeline->bytes_collected); @@ -316,7 +316,7 @@ static ssize_t kbasep_timeline_io_read(struct file *filp, char __user *buffer, s } else { const unsigned int new_rb_idx_raw = wb_idx_raw - PACKET_COUNT + 1; /* Adjust read buffer index to the next valid buffer */ - atomic_set(&stream->rbi, new_rb_idx_raw); + atomic_set(&stream->rbi, (int)new_rb_idx_raw); } } diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c index ddbddcbc5968..117417c30183 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tlstream.c @@ -222,9 +222,9 @@ char *kbase_tlstream_msgbuf_acquire(struct kbase_tlstream *stream, size_t msg_si spin_lock_irqsave(&stream->lock, *flags); - wb_idx_raw = atomic_read(&stream->wbi); + wb_idx_raw = (unsigned int)atomic_read(&stream->wbi); wb_idx = wb_idx_raw % PACKET_COUNT; - wb_size = atomic_read(&stream->buffer[wb_idx].size); + wb_size = (size_t)atomic_read(&stream->buffer[wb_idx].size); /* Select next buffer if data will not fit into current one. */ if (wb_size + msg_size > PACKET_SIZE) { @@ -264,9 +264,9 @@ size_t kbase_tlstream_flush_stream(struct kbase_tlstream *stream) spin_lock_irqsave(&stream->lock, flags); - wb_idx_raw = atomic_read(&stream->wbi); + wb_idx_raw = (unsigned int)atomic_read(&stream->wbi); wb_idx = wb_idx_raw % PACKET_COUNT; - wb_size = atomic_read(&stream->buffer[wb_idx].size); + wb_size = (size_t)atomic_read(&stream->buffer[wb_idx].size); if (wb_size > min_size) { wb_size = kbasep_tlstream_msgbuf_submit(stream, wb_idx_raw, wb_size); diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h index 8e09c286a066..b2cbfe6e528d 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h @@ -884,7 +884,7 @@ struct kbase_tlstream; tgid \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_new_ctx( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -909,7 +909,7 @@ struct kbase_tlstream; core_count \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_new_gpu( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -934,7 +934,7 @@ struct kbase_tlstream; lpu_fn \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_new_lpu( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -957,7 +957,7 @@ struct kbase_tlstream; atom_nr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_new_atom( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -979,7 +979,7 @@ struct kbase_tlstream; as_nr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_new_as( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -999,7 +999,7 @@ struct kbase_tlstream; ctx \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_del_ctx( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1018,7 +1018,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_del_atom( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1039,7 +1039,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_lifelink_lpu_gpu( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1061,7 +1061,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_lifelink_as_gpu( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1083,7 +1083,7 @@ struct kbase_tlstream; lpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_ret_ctx_lpu( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1105,7 +1105,7 @@ struct kbase_tlstream; ctx \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_ret_atom_ctx( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1129,7 +1129,7 @@ struct kbase_tlstream; attrib_match_list \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_ret_atom_lpu( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1152,7 +1152,7 @@ struct kbase_tlstream; lpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_nret_ctx_lpu( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1174,7 +1174,7 @@ struct kbase_tlstream; ctx \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_nret_atom_ctx( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1196,7 +1196,7 @@ struct kbase_tlstream; lpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_nret_atom_lpu( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1218,7 +1218,7 @@ struct kbase_tlstream; ctx \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_ret_as_ctx( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1240,7 +1240,7 @@ struct kbase_tlstream; ctx \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_nret_as_ctx( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1262,7 +1262,7 @@ struct kbase_tlstream; address_space \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_ret_atom_as( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1284,7 +1284,7 @@ struct kbase_tlstream; address_space \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_nret_atom_as( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1310,7 +1310,7 @@ struct kbase_tlstream; config \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_attrib_atom_config( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1334,7 +1334,7 @@ struct kbase_tlstream; j_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_jit_usedpages( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1370,7 +1370,7 @@ struct kbase_tlstream; usg_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_attrib_atom_jitallocinfo( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1399,7 +1399,7 @@ struct kbase_tlstream; j_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_attrib_atom_jitfreeinfo( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1425,7 +1425,7 @@ struct kbase_tlstream; transcfg \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_attrib_as_config( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1447,7 +1447,7 @@ struct kbase_tlstream; lpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_event_lpu_softstop( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1466,7 +1466,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_event_atom_softstop_ex( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1485,7 +1485,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_event_atom_softstop_issue( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1504,7 +1504,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_event_atom_softjob_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1523,7 +1523,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_event_atom_softjob_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1542,7 +1542,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_arbiter_granted( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1561,7 +1561,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_arbiter_started( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1580,7 +1580,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_arbiter_stop_requested( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1599,7 +1599,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_arbiter_stopped( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1618,7 +1618,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_arbiter_requested( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1637,7 +1637,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_jd_gpu_soft_reset( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1660,7 +1660,7 @@ struct kbase_tlstream; chunk_va \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_jd_tiler_heap_chunk_alloc( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1681,7 +1681,7 @@ struct kbase_tlstream; dummy \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_js_sched_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1700,7 +1700,7 @@ struct kbase_tlstream; dummy \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_js_sched_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1719,7 +1719,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_jd_submit_atom_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1738,7 +1738,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_jd_submit_atom_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1757,7 +1757,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_jd_done_no_lock_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1776,7 +1776,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_jd_done_no_lock_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1795,7 +1795,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_jd_done_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1814,7 +1814,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_jd_done_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1833,7 +1833,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_jd_atom_complete( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1854,7 +1854,7 @@ struct kbase_tlstream; atom_nr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_run_atom_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1876,7 +1876,7 @@ struct kbase_tlstream; atom_nr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_tl_run_atom_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1898,7 +1898,7 @@ struct kbase_tlstream; prio \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_tl_attrib_atom_priority( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1920,7 +1920,7 @@ struct kbase_tlstream; state \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_tl_attrib_atom_state( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1940,7 +1940,7 @@ struct kbase_tlstream; atom \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_tl_attrib_atom_prioritized( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -1975,7 +1975,7 @@ struct kbase_tlstream; va_pgs \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_JOB_DUMPING_ENABLED) \ __kbase_tlstream_tl_attrib_atom_jit( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2017,7 +2017,7 @@ struct kbase_tlstream; kbase_device_has_vd54d34dbb40917c8cea48cca407a8789413be0db \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_new_device( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2060,7 +2060,7 @@ struct kbase_tlstream; buffer_gpu_addr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_gpucmdqueue_kick( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2097,7 +2097,7 @@ struct kbase_tlstream; kbase_device_csg_slot_resuming \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_device_program_csg( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2134,7 +2134,7 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_device_deprogram_csg( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2167,7 +2167,7 @@ struct kbase_tlstream; kbase_device_csg_slot_suspending \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_device_halting_csg( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2200,7 +2200,7 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_device_suspend_csg( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2231,7 +2231,7 @@ struct kbase_tlstream; kbase_device_csg_slot_index \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_device_csg_idle( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2262,7 +2262,7 @@ struct kbase_tlstream; kbase_device_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_new_ctx( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2291,7 +2291,7 @@ struct kbase_tlstream; kernel_ctx_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_del_ctx( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2320,7 +2320,7 @@ struct kbase_tlstream; kbase_device_as_index \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_ctx_assign_as( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2349,7 +2349,7 @@ struct kbase_tlstream; kernel_ctx_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_ctx_unassign_as( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2382,7 +2382,7 @@ struct kbase_tlstream; kcpuq_num_pending_cmds \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_new_kcpuqueue( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2415,7 +2415,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_del_kcpuqueue( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2444,7 +2444,7 @@ struct kbase_tlstream; fence \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_signal( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2475,7 +2475,7 @@ struct kbase_tlstream; fence \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2510,7 +2510,7 @@ struct kbase_tlstream; inherit_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2545,7 +2545,7 @@ struct kbase_tlstream; cqs_obj_gpu_addr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2584,7 +2584,7 @@ struct kbase_tlstream; inherit_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait_operation( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2629,7 +2629,7 @@ struct kbase_tlstream; data_type \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set_operation( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2666,7 +2666,7 @@ struct kbase_tlstream; map_import_buf_gpu_addr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2697,7 +2697,7 @@ struct kbase_tlstream; map_import_buf_gpu_addr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2728,7 +2728,7 @@ struct kbase_tlstream; map_import_buf_gpu_addr \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2757,7 +2757,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2802,7 +2802,7 @@ struct kbase_tlstream; jit_alloc_usage_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_alloc( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2847,7 +2847,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_alloc( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2874,7 +2874,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_free( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2903,7 +2903,7 @@ struct kbase_tlstream; jit_alloc_jit_id \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_free( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2932,7 +2932,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_free( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2959,7 +2959,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_error_barrier( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -2990,7 +2990,7 @@ struct kbase_tlstream; gpu_cmdq_grp_handle \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_group_suspend( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3021,7 +3021,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3050,7 +3050,7 @@ struct kbase_tlstream; execute_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3079,7 +3079,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3108,7 +3108,7 @@ struct kbase_tlstream; execute_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3137,7 +3137,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3166,7 +3166,7 @@ struct kbase_tlstream; execute_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3197,7 +3197,7 @@ struct kbase_tlstream; execute_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3226,7 +3226,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_operation_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3255,7 +3255,7 @@ struct kbase_tlstream; execute_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_operation_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3286,7 +3286,7 @@ struct kbase_tlstream; execute_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set_operation( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3315,7 +3315,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3344,7 +3344,7 @@ struct kbase_tlstream; execute_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3373,7 +3373,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3402,7 +3402,7 @@ struct kbase_tlstream; execute_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3431,7 +3431,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3460,7 +3460,7 @@ struct kbase_tlstream; execute_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3489,7 +3489,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3516,7 +3516,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3549,7 +3549,7 @@ struct kbase_tlstream; jit_alloc_mmu_flags \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3582,7 +3582,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3609,7 +3609,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_free_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3636,7 +3636,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3667,7 +3667,7 @@ struct kbase_tlstream; jit_free_pages_used \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3698,7 +3698,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3725,7 +3725,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_error_barrier( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3752,7 +3752,7 @@ struct kbase_tlstream; kcpu_queue \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_group_suspend_start( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3781,7 +3781,7 @@ struct kbase_tlstream; execute_error \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_kcpuqueue_execute_group_suspend_end( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3810,7 +3810,7 @@ struct kbase_tlstream; csffw_cycle \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_csffw_fw_reloading( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3837,7 +3837,7 @@ struct kbase_tlstream; csffw_cycle \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_csffw_fw_enabling( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3864,7 +3864,7 @@ struct kbase_tlstream; csffw_cycle \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_csffw_fw_request_sleep( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3891,7 +3891,7 @@ struct kbase_tlstream; csffw_cycle \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_csffw_fw_request_wakeup( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3918,7 +3918,7 @@ struct kbase_tlstream; csffw_cycle \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_csffw_fw_request_halt( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3945,7 +3945,7 @@ struct kbase_tlstream; csffw_cycle \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_csffw_fw_disabling( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -3972,7 +3972,7 @@ struct kbase_tlstream; csffw_cycle \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_csffw_fw_off( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -4001,7 +4001,7 @@ struct kbase_tlstream; csffw_cycle \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( \ __TL_DISPATCH_STREAM(kbdev, obj), \ @@ -4031,7 +4031,7 @@ struct kbase_tlstream; core_state_bitset \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_pm_state( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4055,7 +4055,7 @@ struct kbase_tlstream; page_cnt_change \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_pagefault( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4078,7 +4078,7 @@ struct kbase_tlstream; page_cnt \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_pagesalloc( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4098,7 +4098,7 @@ struct kbase_tlstream; target_freq \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_devfreq_target( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4127,7 +4127,7 @@ struct kbase_tlstream; ph_pages \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_jit_stats( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4167,7 +4167,7 @@ struct kbase_tlstream; nr_in_flight \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_tiler_heap_stats( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4200,7 +4200,7 @@ struct kbase_tlstream; event \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_event_job_slot( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4222,7 +4222,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_protected_enter_start( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4241,7 +4241,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_protected_enter_end( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4268,7 +4268,7 @@ struct kbase_tlstream; mmu_lock_page_num \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_mmu_command( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4291,7 +4291,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_aux_protected_leave_start( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4310,7 +4310,7 @@ struct kbase_tlstream; gpu \ ) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ if (enabled & BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS) \ __kbase_tlstream_aux_protected_leave_end( \ __TL_DISPATCH_STREAM(kbdev, aux), \ @@ -4332,7 +4332,7 @@ struct kbase_tlstream; #define KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, \ context, slot_nr, atom_nr, event) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ kbase_trace_mali_job_slots_event(kbdev->id, \ GATOR_MAKE_EVENT(event, slot_nr), \ context, (u8) atom_nr); \ @@ -4345,7 +4345,7 @@ struct kbase_tlstream; #undef KBASE_TLSTREAM_AUX_PM_STATE #define KBASE_TLSTREAM_AUX_PM_STATE(kbdev, core_type, state) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ kbase_trace_mali_pm_status(kbdev->id, \ core_type, state); \ if (enabled & TLSTREAM_ENABLED) \ @@ -4358,9 +4358,9 @@ struct kbase_tlstream; #define KBASE_TLSTREAM_AUX_PAGEFAULT(kbdev, \ ctx_nr, as_nr, page_cnt_change) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ kbase_trace_mali_page_fault_insert_pages(kbdev->id, \ - as_nr, \ + (int)as_nr, \ page_cnt_change); \ if (enabled & TLSTREAM_ENABLED) \ __kbase_tlstream_aux_pagefault( \ @@ -4375,9 +4375,9 @@ struct kbase_tlstream; #undef KBASE_TLSTREAM_AUX_PAGESALLOC #define KBASE_TLSTREAM_AUX_PAGESALLOC(kbdev, ctx_nr, page_cnt) \ do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ u32 global_pages_count = \ - atomic_read(&kbdev->memdev.used_pages); \ + (u32)atomic_read(&kbdev->memdev.used_pages); \ \ kbase_trace_mali_total_alloc_pages_change(kbdev->id, \ global_pages_count); \ diff --git a/include/linux/memory_group_manager.h b/include/linux/memory_group_manager.h index 557853d72718..3820f1bff86b 100644 --- a/include/linux/memory_group_manager.h +++ b/include/linux/memory_group_manager.h @@ -58,13 +58,17 @@ struct memory_group_manager_ops { * 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. * @gfp_mask: Bitmask of Get Free Page flags affecting allocator * behavior. - * @order: Page order for physical page size (order=0 means 4 KiB, - * order=9 means 2 MiB). + * @order: Page order for physical page size. + * order = 0 refers to small pages + * order != 0 refers to 2 MB pages, so + * order = 9 (when small page size is 4KB, 2^9 * 4KB = 2 MB) + * order = 7 (when small page size is 16KB, 2^7 * 16KB = 2 MB) + * order = 5 (when small page size is 64KB, 2^5 * 64KB = 2 MB) * * Return: Pointer to allocated page, or NULL if allocation failed. */ - struct page *(*mgm_alloc_page)(struct memory_group_manager_device *mgm_dev, int group_id, - gfp_t gfp_mask, unsigned int order); + struct page *(*mgm_alloc_page)(struct memory_group_manager_device *mgm_dev, + unsigned int group_id, gfp_t gfp_mask, unsigned int order); /* * mgm_free_page - Free a physical memory page in a group @@ -78,10 +82,11 @@ struct memory_group_manager_ops { * memory that was allocated by calling the mgm_alloc_page * method of the same memory pool with the same values of * @group_id and @order. - * @order: Page order for physical page size (order=0 means 4 KiB, - * order=9 means 2 MiB). + * @order: Page order for physical page size. + * order = 0 refers to small pages + * order != 0 refers to 2 MB pages. */ - void (*mgm_free_page)(struct memory_group_manager_device *mgm_dev, int group_id, + void (*mgm_free_page)(struct memory_group_manager_device *mgm_dev, unsigned int group_id, struct page *page, unsigned int order); /* @@ -124,8 +129,8 @@ struct memory_group_manager_ops { * * Return: A modified GPU page table entry to be stored in a page table. */ - u64 (*mgm_update_gpu_pte)(struct memory_group_manager_device *mgm_dev, int group_id, - int mmu_level, u64 pte); + u64 (*mgm_update_gpu_pte)(struct memory_group_manager_device *mgm_dev, + unsigned int group_id, int mmu_level, u64 pte); /* * mgm_pte_to_original_pte - Undo any modification done during mgm_update_gpu_pte() @@ -145,8 +150,8 @@ struct memory_group_manager_ops { * * Return: PTE entry as originally specified to mgm_update_gpu_pte() */ - u64 (*mgm_pte_to_original_pte)(struct memory_group_manager_device *mgm_dev, int group_id, - int mmu_level, u64 pte); + u64 (*mgm_pte_to_original_pte)(struct memory_group_manager_device *mgm_dev, + unsigned int group_id, int mmu_level, u64 pte); /* * mgm_vmf_insert_pfn_prot - Map a physical page in a group for the CPU @@ -170,7 +175,7 @@ struct memory_group_manager_ops { * table entry was successfully installed. */ vm_fault_t (*mgm_vmf_insert_pfn_prot)(struct memory_group_manager_device *mgm_dev, - int group_id, struct vm_area_struct *vma, + unsigned int group_id, struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot); }; diff --git a/include/linux/priority_control_manager.h b/include/linux/priority_control_manager.h index 9f28b1b8582a..d82419ebf523 100644 --- a/include/linux/priority_control_manager.h +++ b/include/linux/priority_control_manager.h @@ -28,32 +28,78 @@ struct priority_control_manager_device; +/** + * DOC: PCM notifier callback types + * + * ADD_PRIORITIZED_PROCESS - indicate that work items for this process should be + * given priority over the work items from other + * processes that were assigned the same static + * priority level. Processes that would benefit from + * being added to this list includes foreground + * applications, as well as any other latency-sensitive + * applications. + * + * REMOVE_PRIORITIZED_PROCESS - indicate that work items for this process + * should no longer be prioritized over other work + * items given the same static priority level. + */ +#define ADD_PRIORITIZED_PROCESS 0 +#define REMOVE_PRIORITIZED_PROCESS 1 + +/** + * struct pcm_prioritized_process_notifier_data - change of prioritized process + * list passed to the callback + * + * @pid: PID of the process being added/removed + */ +struct pcm_prioritized_process_notifier_data { + uint32_t pid; +}; + /** * struct priority_control_manager_ops - Callbacks for priority control manager operations * * @pcm_scheduler_priority_check: Callback to check if scheduling priority level can be requested + * pcm_dev: The priority control manager through which the + * request is being made. + * task: The task struct of the process requesting the + * priority check. + * requested_priority: The priority level being requested. + * + * The returned value will be: + * The same as requested_priority if the process has permission to + * use requested_priority.A lower priority value if the process does + * not have permission to use requested_priority + * + * requested_priority has the following value range: + * 0-3 : Priority level, 0 being highest and 3 being lowest + * + * Return: The priority that would actually be given, could be lower + * than requested_priority + * + * @pcm_prioritized_process_notifier_register: register a callback for changes to the + * list of prioritized processes + * pcm_dev: The priority control manager through + * which the request is being made. + * nb: notifier block with callback function pointer + * On Success returns 0 otherwise -1 + * + * @pcm_prioritized_process_notifier_unregister: unregister the callback for changes to the + * list of prioritized processes + * pcm_dev: The priority control manager through + * which the request is being made. + * nb: notifier block which will be unregistered + * On Success returns 0 otherwise -1 */ struct priority_control_manager_ops { - /* - * pcm_scheduler_priority_check: This function can be used to check what priority its work - * would be treated as based on the requested_priority value. - * - * @pcm_dev: The priority control manager through which the request is - * being made. - * @task: The task struct of the process requesting the priority check. - * @requested_priority: The priority level being requested. - * - * The returned value will be: - * The same as requested_priority if the process has permission to use requested_priority - * A lower priority value if the process does not have permission to use requested_priority - * - * requested_priority has the following value range: - * 0-3 : Priority level, 0 being highest and 3 being lowest - * - * Return: The priority that would actually be given, could be lower than requested_priority - */ int (*pcm_scheduler_priority_check)(struct priority_control_manager_device *pcm_dev, struct task_struct *task, int requested_priority); + + int (*pcm_prioritized_process_notifier_register)( + struct priority_control_manager_device *pcm_dev, struct notifier_block *nb); + + int (*pcm_prioritized_process_notifier_unregister)( + struct priority_control_manager_device *pcm_dev, struct notifier_block *nb); }; /** diff --git a/include/linux/version_compat_defs.h b/include/linux/version_compat_defs.h index 3f46e852bdc9..366b50c4e3ec 100644 --- a/include/linux/version_compat_defs.h +++ b/include/linux/version_compat_defs.h @@ -25,6 +25,7 @@ #include #include #include +#include #if (KERNEL_VERSION(4, 4, 267) < LINUX_VERSION_CODE) #include @@ -188,6 +189,7 @@ static inline void kbase_kunmap_atomic(void *address) #define dma_fence fence #define dma_fence_ops fence_ops +#define dma_fence_cb fence_cb #define dma_fence_context_alloc(a) fence_context_alloc(a) #define dma_fence_init(a, b, c, d, e) fence_init(a, b, c, d, e) #define dma_fence_get(a) fence_get(a) @@ -297,4 +299,53 @@ static inline long kbase_pin_user_pages_remote(struct task_struct *tsk, struct m #define kbase_totalram_pages() totalram_pages() #endif /* KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE */ +#ifndef read_poll_timeout_atomic +#define read_poll_timeout_atomic(op, val, cond, delay_us, timeout_us, delay_before_read, args...) \ + ({ \ + const u64 __timeout_us = (timeout_us); \ + s64 __left_ns = __timeout_us * NSEC_PER_USEC; \ + const unsigned long __delay_us = (delay_us); \ + const u64 __delay_ns = __delay_us * NSEC_PER_USEC; \ + if (delay_before_read && __delay_us) \ + udelay(__delay_us); \ + if (__timeout_us) \ + __left_ns -= __delay_ns; \ + do { \ + (val) = op(args); \ + if (__timeout_us) { \ + if (__delay_us) { \ + udelay(__delay_us); \ + __left_ns -= __delay_ns; \ + } \ + __left_ns--; \ + } \ + } while (!(cond) && (!__timeout_us || (__left_ns > 0))); \ + (cond) ? 0 : -ETIMEDOUT; \ + }) +#endif + +#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) + +#define kbase_refcount_t atomic_t +#define kbase_refcount_read(x) atomic_read(x) +#define kbase_refcount_set(x, v) atomic_set(x, v) +#define kbase_refcount_dec_and_test(x) atomic_dec_and_test(x) +#define kbase_refcount_dec(x) atomic_dec(x) +#define kbase_refcount_inc_not_zero(x) atomic_inc_not_zero(x) +#define kbase_refcount_inc(x) atomic_inc(x) + +#else + +#include + +#define kbase_refcount_t refcount_t +#define kbase_refcount_read(x) refcount_read(x) +#define kbase_refcount_set(x, v) refcount_set(x, v) +#define kbase_refcount_dec_and_test(x) refcount_dec_and_test(x) +#define kbase_refcount_dec(x) refcount_dec(x) +#define kbase_refcount_inc_not_zero(x) refcount_inc_not_zero(x) +#define kbase_refcount_inc(x) refcount_inc(x) + +#endif /* (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) */ + #endif /* _VERSION_COMPAT_DEFS_H_ */ diff --git a/include/uapi/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.h b/include/uapi/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.h index 46627c416baa..56a16e11c4c0 100644 --- a/include/uapi/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.h +++ b/include/uapi/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.h @@ -62,7 +62,7 @@ struct dma_buf_te_ioctl_set_failing { struct dma_buf_te_ioctl_fill { int fd; - unsigned int value; + int value; }; #define DMA_BUF_TE_IOCTL_BASE 'E' diff --git a/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h b/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h index b45e32fc3d33..564f477e57d1 100644 --- a/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h +++ b/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.h @@ -36,7 +36,7 @@ #endif /* MALI_USE_CSF */ #define KBASE_DUMMY_MODEL_COUNTERS_PER_BIT (4) #define KBASE_DUMMY_MODEL_COUNTER_ENABLED(enable_mask, ctr_idx) \ - (enable_mask & (1 << (ctr_idx / KBASE_DUMMY_MODEL_COUNTERS_PER_BIT))) + (enable_mask & (1U << (ctr_idx / KBASE_DUMMY_MODEL_COUNTERS_PER_BIT))) #define KBASE_DUMMY_MODEL_HEADERS_PER_BLOCK 4 #define KBASE_DUMMY_MODEL_COUNTERS_PER_BLOCK KBASE_DUMMY_MODEL_COUNTER_PER_CORE diff --git a/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h b/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h index c83cedd6a775..7e56fd71f84e 100644 --- a/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h +++ b/include/uapi/gpu/arm/bifrost/backend/gpu/mali_kbase_model_linux.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,7 +30,6 @@ #define MODEL_LINUX_JOB_IRQ (0x1 << 0) #define MODEL_LINUX_GPU_IRQ (0x1 << 1) #define MODEL_LINUX_MMU_IRQ (0x1 << 2) - #define MODEL_LINUX_IRQ_MASK (MODEL_LINUX_JOB_IRQ | MODEL_LINUX_GPU_IRQ | MODEL_LINUX_MMU_IRQ) #endif /* _UAPI_KBASE_MODEL_LINUX_H_ */ diff --git a/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h b/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h index 013924887142..0fb824267184 100644 --- a/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h +++ b/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h @@ -96,10 +96,10 @@ /* Flags for base tracepoint specific to CSF */ /* Enable KBase tracepoints for CSF builds */ -#define BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS (1 << 2) +#define BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS (1U << 2) /* Enable additional CSF Firmware side tracepoints */ -#define BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS (1 << 3) +#define BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS (1U << 3) #define BASE_TLSTREAM_FLAGS_MASK \ (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | BASE_TLSTREAM_JOB_DUMPING_ENABLED | \ @@ -141,7 +141,7 @@ #define BASE_CSF_EXCEPTION_HANDLER_FLAGS_MASK (BASE_CSF_TILER_OOM_EXCEPTION_FLAG) /* Initial value for LATEST_FLUSH register */ -#define POWER_DOWN_LATEST_FLUSH_VALUE ((uint32_t)1) +#define POWER_DOWN_LATEST_FLUSH_VALUE ((__u32)1) /** * enum base_kcpu_command_type - Kernel CPU queue command type. @@ -157,7 +157,7 @@ * @BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: jit_alloc, * @BASE_KCPU_COMMAND_TYPE_JIT_FREE: jit_free, * @BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: group_suspend, - * @BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: error_barrier, + * @BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: error_barrier */ enum base_kcpu_command_type { BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL, @@ -172,7 +172,7 @@ enum base_kcpu_command_type { BASE_KCPU_COMMAND_TYPE_JIT_ALLOC, BASE_KCPU_COMMAND_TYPE_JIT_FREE, BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND, - BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER, + BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER }; /** diff --git a/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h b/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h index 9db2146e2fd5..537c90d6efa5 100644 --- a/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h +++ b/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h @@ -95,10 +95,15 @@ * 1.22: * - Add comp_pri_threshold and comp_pri_ratio attributes to * kbase_ioctl_cs_queue_group_create. + * 1.23: + * - Disallows changing the sharability on the GPU of imported dma-bufs to + * BASE_MEM_COHERENT_SYSTEM using KBASE_IOCTL_MEM_FLAGS_CHANGE. + * 1.24: + * - Implement full block state support for hardware counters. */ #define BASE_UK_VERSION_MAJOR 1 -#define BASE_UK_VERSION_MINOR 22 +#define BASE_UK_VERSION_MINOR 24 /** * struct kbase_ioctl_version_check - Check version compatibility between diff --git a/include/uapi/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h b/include/uapi/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h index de392a5c506f..ee64184e2794 100644 --- a/include/uapi/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h +++ b/include/uapi/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h @@ -22,9 +22,9 @@ #ifndef _UAPI_KBASE_GPU_COHERENCY_H_ #define _UAPI_KBASE_GPU_COHERENCY_H_ -#define COHERENCY_ACE_LITE 0 -#define COHERENCY_ACE 1 -#define COHERENCY_NONE 31 -#define COHERENCY_FEATURE_BIT(x) (1 << (x)) +#define COHERENCY_ACE_LITE 0U +#define COHERENCY_ACE 1U +#define COHERENCY_NONE 31U +#define COHERENCY_FEATURE_BIT(x) (1U << (x)) #endif /* _UAPI_KBASE_GPU_COHERENCY_H_ */ diff --git a/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h b/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h index 1827d6ec4e1b..2a7a06a995be 100644 --- a/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h +++ b/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h @@ -149,10 +149,15 @@ * from the parent process. * 11.40: * - Remove KBASE_IOCTL_HWCNT_READER_SETUP and KBASE_HWCNT_READER_* ioctls. + * 11.41: + * - Disallows changing the sharability on the GPU of imported dma-bufs to + * BASE_MEM_COHERENT_SYSTEM using KBASE_IOCTL_MEM_FLAGS_CHANGE. + * 11.42: + * - Implement full block state support for hardware counters. */ #define BASE_UK_VERSION_MAJOR 11 -#define BASE_UK_VERSION_MINOR 40 +#define BASE_UK_VERSION_MINOR 42 /** * struct kbase_ioctl_version_check - Check version compatibility between diff --git a/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h b/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h index 82e651f67b71..c009d5ddd494 100644 --- a/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h +++ b/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h @@ -221,11 +221,11 @@ typedef __u32 base_context_create_flags; /* Enable additional tracepoints for latency measurements (TL_ATOM_READY, * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) */ -#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) +#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1U << 0) /* Indicate that job dumping is enabled. This could affect certain timers * to account for the performance impact. */ -#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) +#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1U << 1) #endif /* _UAPI_BASE_COMMON_KERNEL_H_ */ diff --git a/include/uapi/gpu/arm/bifrost/mali_base_kernel.h b/include/uapi/gpu/arm/bifrost/mali_base_kernel.h index 8e507f0f14aa..cb1a1e8dd550 100644 --- a/include/uapi/gpu/arm/bifrost/mali_base_kernel.h +++ b/include/uapi/gpu/arm/bifrost/mali_base_kernel.h @@ -32,20 +32,26 @@ #include "gpu/mali_kbase_gpu_id.h" #include "gpu/mali_kbase_gpu_coherency.h" +#ifdef __KERNEL__ +#include + #if defined(PAGE_MASK) && defined(PAGE_SHIFT) #define LOCAL_PAGE_SHIFT PAGE_SHIFT #define LOCAL_PAGE_LSB ~PAGE_MASK #else -#ifndef OSU_CONFIG_CPU_PAGE_SIZE_LOG2 -#define OSU_CONFIG_CPU_PAGE_SIZE_LOG2 12 +#error "Missing kernel definitions: PAGE_MASK, PAGE_SHIFT" #endif -#if defined(OSU_CONFIG_CPU_PAGE_SIZE_LOG2) -#define LOCAL_PAGE_SHIFT OSU_CONFIG_CPU_PAGE_SIZE_LOG2 -#define LOCAL_PAGE_LSB ((1ul << OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - 1) #else -#error Failed to find page size + +#if defined(MALI_PAGE_SIZE_AGNOSTIC) +#define LOCAL_PAGE_SHIFT (__builtin_ctz((unsigned int)sysconf(_SC_PAGESIZE))) +#else +#define LOCAL_PAGE_SHIFT 12 #endif + +#define LOCAL_PAGE_LSB ((1ul << LOCAL_PAGE_SHIFT) - 1) + #endif /* Physical memory group ID for normal usage. diff --git a/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h b/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h index 329845005341..11c51d9c2993 100644 --- a/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h +++ b/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014, 2017-2022 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,6 +30,6 @@ * KBASE_MEM_PROFILE_MAX_BUF_SIZE - The size of the buffer to accumulate the histogram report text * in @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT */ -#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t)(64 + ((80 + (56 * 64)) * 54) + 56)) +#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t)(64 + ((80 + (56 * 64)) * 57) + 56)) #endif /*_UAPI_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/ From 386decb92ee0c8543db5e4397a01d06a1dc3c9de Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Sat, 11 May 2024 10:25:47 +0800 Subject: [PATCH 4/6] MALI: bifrost: CSF: Using DDK version as prefix in 'default_fw_name' Change-Id: I7e4ab0323794cdc4db8789e4aef317c357dc4938 Signed-off-by: Zhen Chen --- drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c index 8ad1d74b64d5..8e4608c7e528 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c @@ -55,9 +55,11 @@ #include #include -#define MALI_MAX_DEFAULT_FIRMWARE_NAME_LEN ((size_t)20) +#define MALI_MAX_DEFAULT_FIRMWARE_NAME_LEN ((size_t)64) -static char default_fw_name[MALI_MAX_DEFAULT_FIRMWARE_NAME_LEN] = "mali_csffw.bin"; +#define DEFAULT_FW_NAME MALI_RELEASE_NAME".mali_csffw.bin" + +static char default_fw_name[MALI_MAX_DEFAULT_FIRMWARE_NAME_LEN] = DEFAULT_FW_NAME; module_param_string(fw_name, default_fw_name, sizeof(default_fw_name), 0644); MODULE_PARM_DESC(fw_name, "firmware image"); @@ -2345,6 +2347,7 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) #endif /* IS_ENABLED(CONFIG_OF) */ + dev_info(kbdev->dev, "to load firmware image '%s'\n", fw_name); if (request_firmware(&firmware, fw_name, kbdev->dev) != 0) { dev_err(kbdev->dev, "Failed to load firmware image '%s'\n", fw_name); ret = -ENOENT; From d6fdcb0bf4c2d0fef99570f1786ea3e91557ba25 Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Wed, 17 Jul 2024 16:17:42 +0800 Subject: [PATCH 5/6] MALI: bifrost: CSF: Add formal mali_csffw.bin of Valhall DDK g22(r47) Change-Id: Ia97d5d1f597dbeda65ec78431304ec0e851d9eed Signed-off-by: Zhen Chen --- drivers/gpu/arm/bifrost/mali_csffw.bin | Bin 0 -> 274432 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 drivers/gpu/arm/bifrost/mali_csffw.bin diff --git a/drivers/gpu/arm/bifrost/mali_csffw.bin b/drivers/gpu/arm/bifrost/mali_csffw.bin new file mode 100644 index 0000000000000000000000000000000000000000..1f8413ba14d7310dd3c6b254e16c3244305b1980 GIT binary patch literal 274432 zcmeFadwf*Yxj(%2o;}yxCgBz!!k&c7a8ZYw25Re`nPewi6GG5vwM>A>M7azXC7$C? zb`UGoUIL;GXf1lQ#nwtNowYaQ;mj zLU(Sf605gWZK>S0X^Xh^&aI)UjcdiN+bY*piCgce4v8Box86bdsjM)lth}Xi-G-`- z+y0fpTs~85gUXgyudmwjFBIfN*4&e2z)SsGvT2=oSJjr)o3>VOyPIIjW=d+Z-hTXU z^se&DMIL5yQ$HDVB5fpGHe3#z2uHu$w%@t!HuxqFQ#kk(KMBqaHxteSNBFv}ddr6G zm0POBZCfhWR3X2cHB#H}Sb>y9Ew7(wdA-HXDHE=fM zCHh%gwffF=;)YEdt7$N+Hx4dy5ArE+yWn=irCzqoraQM$J4u+ni+pu(`{3%~oJpA0 zh3;Inp=#Tf>NQ)L)yb68XG%Jp|v>BO>i z;Z`;QnJc%fTeY!rL)A>mwn|SoJ}FWMfzWZWksAt^m9JV@URbtj;j)G03%yHL`Ae29 zEA*EyELpZfN8Q4K=~=y1T(fE8#;P^js@96zHi@@wTEBkN_Ueu602o@QDJ?Boh@W0; z&E0F(SFKvJ>CTPYX3`AT|IjpFwe9Xu)yy?p*TGe9B`mOsmx2TwtSI-E7q0RbdzTdz z>WGtXB-Q|qu_jo#ah=g`w8%!G_~7Pl+rCQAr*oxom)7lHv1;WC^tiys@{sB!_{Cu5 zR&h(!nyTu%(7Q_U&aL1PYd3A$vbt*hdS*s8>KnsQ4r$8hhCTw9=>!_Oq!a!XLpxzf zR=p5B;&OEZOUd345F*^L!Sk;u8rll+0z?jN=0Xn$AA^x)^k7}G4T6Sna!DJ-H!hTi zHZxMsM)5;i8bk-vFKH{dz52G9+t#mIvz5M#Mw7Il6A;n+u(oyL9D>Ht$aH}!hfu_z zwr3A(`!Y00Q94EHbr2W~Q}=Szh=KI_m#P(?ORDE>A#Zg-Z&(D+aY&R<*vWa_j$6y8w`p%zz=Ge&}O(!~lr&FVmV_<}F$Z26g#@VC`(k z?7j%*vci>R-X(f5Fj(r{1)e~WG<+AD1E*P5QnnPFZ)r(^0psv8mvjwfO3O;h!6pj} zzPcFojCj-FrdBK|EG%8PtO(JJ5#I+BOwK)u$$~SogOo7eN=i!$3n*b&UV?~t(XhOi zuXrdNdc`YOlol>4xO{fPD^0PBbz^y0x^S7d{POv+gbo^bs@}Jx#J?Dr0gj5w3RkQ^ zhT$C`esqyq1{W$QlncwsF57G3PM0r)R!UZu`HkTlidNzYSHNBD&LHs0m#!j2AbB`| z#0f;WON*^2_5!?BD}1X83jN42yd=r^w9a2FDSKBgF^G65XcQXSpA{DdX4O{}fSw zHdd}5M89K@!{|387+RSM7Ontj1B63lb@{?X&=-j=S-6n8_0P)^eGi$^`jU77lSutX zU=z6y;dWGdIK_W8;6(i-cn#?_F^8qAE)FKr{}Bi@8HO+l0zvP5avFSf^E4$c!JC2f zWLX+W9p@KUT*O?`XMKVHrv+)4hM`(7I2hQ=HAq7=j5hRg#oi@gp#+XjaKquegvI+; z41#ljGA`F34a?;KE%mNk@sG;UP+SgBDwhP5=9IozlGt$PR)iS#c>@%nFG54`I6VJQ z-mT{!jK1O733bD&A2Kh7=O>(9TpIN12X}gSc0$y!>@+9?I2xXPD5(zW>_7n%e|)0C z?iVk0Xy_mQMlprG2}i$M-3nVVU18U9Mm>9GDC`b6jvwlNjkuJV3cD5V|0}S$wOV#SXzpq!=oVyiv-@n66M;(L84dnU%{eO&sJm8jo3+oj2 z$afUB4!(bn!j{5)Z?D4k!`(SvVV}T%f1i<8FYv$c$3L-_y_4c){|T3!>SaHNZ-@UX z{7$$7aBk>D7QxMd8~71VBS!VF{)nv%l=0x(4L8#$JP<839On5z1J64fdHxx(mVE@+ zh}F}QM?%niz%>i*dbt0STg&#qJp*UXSJ;1DtFU&sfge>$KS}?4@sF>hq+IluEGcj> z{qsg|*;3IfWBC_LX*)0|+QBAo6^Y?_%S+0{$!kUEL)KPrtz5mnYOMpyKP^mO8M~!w zUG-K-OX6)%K3=Y-$~D`dc^F#HK%70f>MioFSXl6NPt9acjf2^B-m43Vt5*0+$_jlY z-m;6S$s&b@ey41CQk=L@dPq7k{IAHMx02kyT_~K4TqZ9px)8V2zY_Ak?1wrD8l=G~ zdTt2K1*CE*DJk<6F2P>QyJGQ$vU-k%0CGqy)mT)za@8dXN!THPuP9t{p_0L|tI8iIppcq+Q$|4oe&Wb7+yv;%HdO z3s@Q!pRDoXP!CHV0G04KEXingC@9J5XlpP$`;eA&q8V0SGDDIoht{jNb=i~w0EbmJ z5KVBGd2giD3n)%DWYn3|5?n~sW3j6%Em;U^)bTPjZTZT=mC!7ftXy`nMgtYYs1(f( zEt!PnLgWyPT#|cWqX|ld7NhK@fnsHz1)@mvIKM_Ln>FZr_Mq=M7rq^=Z0?1CST=9a z_cepQuf6b%0<%z6vV|3;7ZSuJWf#9o{TIGbApb%@ESo>*dlq2504iWLzQKQzT`w>5 zmeRS$D$lAVC3*+fuP632JbBhXPM-abljmGE*}oJzhVn8Xw_s>`lG~KtSmuQ$`-Z}y zsY%cVN?#7_!lf&hK;Kca3~KQeXkkd6i?yTk%T^3c9H3gA(!B)*r2iP2BiXz$6)u}& z&{Vl>`Tzi^o7DG1fE!Bu-X%*)ZXB8+S-7mwTX56Rl%Yj0%^(i^7*Xy5krmdgsS43S z!~zj(Badq~d>%|gT)(`i-&Q=8J z#AQ%~aXxoRhV7fKE-POtmMj}uZu`*Kf$9fxB`X^UCL0TF+Kwi5HF&bN0Trd8;Jl!C zr1dAcbgaV2vf|41+i>E+5HFTeB}q}@>;HE@$7PF&x-LO1*%aF95-*$M^5rg@bx7m| z!N(E+PeyG`=FVV8nUxVbHC}M4h9G`uQ_S zRQDi%{?)bmc{SAHtQ^}(3L9tF3cD_!)5FQLXg=J?>tX$Tn9hpP2IZ%-Xr%G^Q*m~N zuzq$dBK@Ed{&1SYp8dJPK8JtBGYZpaFZc_EofQ;z>`8^aWH#bY!e94Og_T;2`0v7} z@R?-zQLDm^Kc%n$BM*9cgQ+uh`8{ty;6wZ~5diTL6_+N!j@z?o`a2b3Gzn1i`^BeUh z;8T1~fl>Z8_!Leg{RxFed@+2AuSxo!CjE5Th%bOo{dy4oFVN4=;S(I|i;VPd6&vA4 z;Zr@&CH>Enepk}ZU1+2qf=_s9SY(v@9egV1TVaH6fKTBM%Z+w_wbJlM-(Zw0g-`H= zlK$RgeDO_2{CfCQp1;`$=YP}i_rlM3T46tN811ydr+UVv82R5y`bTat(%*wm^-Q?c z2ycW>`S&LML+~m6FUfFQGCcBIM*I@^lz(?J{CYBcT|i+s{X}7Rtv2f2xz_NXfKTRVhM!7?8r={Juw;v2>p{%i0leeMp0J^8%C_T6Xr ziKM@6ml2l6D{Og#!XE#w!k&3fVc+_m!rnl5F=&L>VFhI z)t|H92zNbZ_cBv*rMMU@gKt1<9}$P zeGE^6R;6Z~`d~MVN{S|4qp;K9)e}@9BfBY7^!IZ%<;s@U5>s)r#==?Z!`=1eT~7DP3Vl2sEmb*>_IMId_`JVRx_Lt z`roPL$`-Z6bhx%*y}aohFFYgr6O$}UJ2d;!j-C{><2aFEmgWRM4Q|FCHJg&%#;(r@ zm$WgDC-8yBq>nUK`nJXna%x+rD4SfX7MlRk%w|&E;RvgDw2F?Mg|DBwd_P!|(UTusz+FLtrT+b# zSz)e&t=TDnJN4ZbTjskhmdw<6idYg$nH<)h4hWBoZet~q+N*@jtaVJSYBoR0T0ONP zCbiyMYmTSdKhWxxY}J;*)DyK#e77Z)z1!koi#k|{@WkRN&agJ};0Ibj{6Jgl$w^mc zvcxe(O1ug@!m>%56kt~|^;gQ`j(c>>vKr2PMdbRKqE$4zfD`6E)WmhM z_Tr8&0$i8S+MKwa9cp4OwRr7B(NZ~Goz+<(r>R@I2jXwO;7Nap)Jf zI8iujaiW;wbQ-bOu;ANPM_p!Qv3q>vfCgzQ!8oZSSO#cZ&Tii6_}OxKgltwH*D|7x zXG`%lx5uHkd0wM7^Se!YyhUB0*~GWC$JwgbTOmpH)n=>FO_J)bEm0MT;>}NZEUfi2 zg+`aTh)xMIEd004rQYMr#nOY~Y z%nD&zPb8cj*6N)Vps5aR{Fg#Ka&S)K6r8OftQ~Z^RE2AXKd48}!s@@?AP@Se8>0c-k6qY)np5Txwh$3iAEzbrpj-Pc2;;+A`kTG^@UiT4vc>(Fump@zI?wCl1?VivurLld8b6Tj9O zDcw}k(Z6|{yxaetR=|gVfoPwIzkL5bt(fPYAk0ZB$95q!_ntfN5kafQ7=ByJRL~o9 zs2cq+b2a`N@Q~8XZl)4HAztFR#dD9@MYuBUH-oN@Yod}W3kNbQ80hO0^pP+=r9Vx5 ztOxBFv+uu*(YZw)=)Y7AE(Q+o$ONxNy}Zj_!%Z_q9M787vz=E&!K1+$)-tEcWn_jo zsV0t#I|I!xnB9&uA82(*+vnu4cEPH%m9e+AeN0Twn@E>a{15pFYH_paah5wz{wB5R z;^HZp4{qbknWvANN1v9=kI7fbJKE4Lzg1k8Kxs!SZH<<;e1#5j*gq0X)nW9>)6B|W&zD^lU z=SqIy4n7&Q1p+}S=<(T_)MFIKxx^Z*CaM=0J^^P7sK>BE7NOn?>ktFa_=zHSNb}fK z8a*}zV=-B*Vew{WjyH>Sn#Wn$7;b6K%JgrEnAPdMoZA%U-RmQ^R?fyFDt%?DJVp~U zoGPDT4KuYDtHv%(#E3c7?w*E1cO<`t#(7?NaY7r#)INqDE~gxXPj`^ZcLzk72Ti$y z*{^USWT7IP`ti$yYK=%LkM((=)^JaLO|vyEg8KAv;ey;OF2F>adb}&YhA2NjgjycB zSi=`dxIXzZ^r9f!Cew@{uCh;?1@2zz)hQ@{7eWisPg6KW{cSg;F$-kYW8jFTg{al! z&W?yRG*f8h6(-Ys!P+}=pf+)b;O3hS?%?tdZWkfbg(N|}^B8yKhzOfPc?=1G_$bTG zRBQXdjnH%AH%49=ZGkepyBY*pxgA(a6*@E<$pnBvOquSJZB0V!ZvJEoyXI!!`nbjX z+i{!Y7AtaE^P9M-?CPARcK?U|5!r=t&f-+98P`Uhs+Ja<9CuIL!p}N2MXp>eW28ix zsn=_$lcl+@jhsRKe{5lSH%XmGZ3>{Bu0uP}cd;hfcW3|k{op-91MuxN zt5+VOx-&H?ku8zTRd+U>H3v;~--BNbDM?UU`t`iE)X%{)!1fBkb6V3XPHR2E6nAFi z;D!oNJYRET%*HHXqNcUI&EqTMHjaxjk-4tSUgj0$Qf>D{U$9!*B4zry2%xC_?Dpd= z>;`c38vu)o;OhUf0i#~0-OtkGX6rcAWmf;av!QVxVu&7h!C#!nrui+hnu^B7iBxfM z!l_FsL>Vk-4+_zSW2ajpDoXdG{-=B){%eEQc^^cAH12Tb#(zTQ1dR;CS<{a587t%gDJ2 zbH+$z@rRhX4jf`uO8ocC5&s>JHFQ5@==~TI6X@|q8WFfWF5c{N^0r&JI6nz8l-t>D z%@^8J*Fl!buh(4csXv_7-U)v1>{HD2C{thUHtC!sGpsdU6JNH5JC~nZe9} zupeV#7iG({g8J7!M=QtKrkTU$dQPp=cFH#QqmdLn&W?T>_~O+2wEK{Tf_#4 zR(F@E{hooexc?I^yP3Hk?)r~&zwElNqeSIQrY_6*qL|tG({et(lSdl@7x!hM9f6O_ zV;;~_-}tHKFxfXcYOL+P&3{>ULSxOD;lHd~7tHqu0=znrnZVz3@=URlywLvrt>gS7 zg4uqS=M22n!e$Xf#-8XQ%tyl8&IZkqmlboEuwzqhmRQ1Z z&1^+dpHRT{b4JmRCnjoxSpQ0s7OH6M5m5-;55oyjA*c3ma{!0}p9`eCQpJtBN- zl(`1>cga?A!kTD8D>=NL@3TI^vnX7?st@Y|1ZPN~8Bxf<2 zxh`AibW1a5^XD`f6sxZh1kZAU#o5h?dVGoLbDa*r_t?fXIrDCe`dw0^sYXsPE2z!- zTTSuI4_Kl?yq1e-rL(!O9mn6yn(eo8xqKJbIyJ`eTUM?fo2AuVKh|dtzwI9@WrmpQ z?=@X1G!ZYV>zR19fK|=I%j$MXjPGY7&JgC-PB2 zq#1e0*Br^3Iv^P(cNHPpRb(c274wJeDl*lsE@M|As42ZqeQC(=gY($22A%C9`TgN9 zi6bbo(~%>OlBJkw-5cjj{uAe{Gm9W4Fg3#h36}w%jKAs|k*n--Ot?#qnfV3&EH#6l z7q+MwR@2q9&hS@X9X9LVGw?kvytJc(C)&!YdDxeG64H1ww6AddNfxgar^|UV$?sC^ zyUZGtNa|Cs(u#k{xhF*?f%hD#*{wC$hzm7WyrrEeCR{rBofRz3VKph8B?XCAzz-4n z>OHWTt0^;X9Gwtk(ypsym8a_L;-9Gra0! z?%J1qGa_v211g8Lf$^q8rriGXfnL-8;v8#OsOP3x!n0nnR5rI*%5yPRu4w;Jnqx)` z!DbIW*@#&-i?!M?i+Ia2K89Oi*?|L zyj;>zU;2lR{!nJM;yD;!;&c|UX$RZKg$}mX3mIW1y3`$hK#sLRRCh@C0rEybe(T4a z=RV)f(2DgiL0@zE09s>!KokOdtNFH2pstPW5SsK-+-CqCsk$oJ#oi7f+ z&4Qt$z~o$%najPf0>_p4D|a{9@+j{(x$k_i*Vbqfr$>cW8^>SyNsH|| zM}3KE=65$0r~+pXe$v9qcQ>&~FT|ZY>|s8Gn?5=`?&!_q?}-b%`6MT%MPuW(#Z&pt zQ*2sEta%IM5tiM=x!81DQ!x`78jcxxTTcEr&A5{L8){|D3iif1>lB)KgcDX%G~w62 zDIeT!e|(ZS(s{yk6fJ^ugx<}H?=E%f9%_qKWePukD8#2 zIQ8v9`Q&)j%KQ<3?EADYa9DyPqo{$m@hHnXEBH9)njhU>IQxU~7~!k+e3qao{(bOs z;UMO$PA9D9#5t31r*GoPV)7OZv)ga4}D%8Xfn`T7GUc72Iet(0_Q zoV;9f&d7rFWR7H7xYNwVM_Fqs#>Et5xuyW=9teY^wx@vgD$loN#CsIRE-?;Xr9lTKJQeR3|q* zB|Q76`M~SI!0Ro7^Ys?X1DxoH60b%Hr$;7U2K+ogbJ#+#*U|{z*TB_`mN8rSi88IG zjCMG*mgZ&b$B6QLJacVHEHjwCKNI^oj21T{T&Epy-hDI-y%BRY#_M+2BiZtPaK<~j zIhnZ^%Q{6xITzr1`C&+(p!h6)j|DVeq6&7?Owc?}6ums~!+;a|e4O|;m8dp5DF3$n zrg^ey=CyKu6PrqTj?ADs{X^X}54m>JyoI7?X+=Nun>nUC0>?rh8@lRk>+^;DBnwPXkp zNrp~oIj5fK?>|4Hi-~_5J`*Mrwdcrt_1QqwZmqd@!t=f>J{Xt*p#FeP{h)o5+aJ51 zKMCqL#mN_x(WhQ3T&B-}#X4oP$(G=lsDo%wUo&pec=6)8Q5e0e@LGKy%pEij3WFuF zaUm%tII;He=IK+;aAKY0RP!=i;R~~&3ZwlrMn&VGsl56O6aRdcQ?KG@6&W*P*nGfT zcnfplonlTLm=QD=j9CGy{imATff+$m4(gt@I9I22tSO-PBc%6r1Sp29;nY5k3kl*E z|1W0Ft28S9`x`R$L$V@S&<5(g3AF1r?MdJ7?6xa)pgew0+J0_N>VCGT|Gc-?EOYK_ zBB|Kv&Pr+mJRy=-F7DA&bDip8eb>w7Rn|ab;l)(wls?Wx zjabWTAt9ROQQAEdAJMLKKZufo`hW($(Rz>BU31Xy4mBJLy?B36kU4$7KV6=qw_F(v z)HEJFbQC?#W$HwZBu_cftn*`EEX?G@#|Cef_+V}fdiwwqIZq%IO4h~?sVz{$ zj6jGqDgR2H*vAJ#$J7^V8!R$_F!Ccb=ZQz%j+!NH7@7}HukL#agaxdmp#z1 zQzotC;*P(L*)0pt@;@VbgPY3Y4(Pl&F+cPOW};ipP*ZzsjZB;nO>1@9`R6}r!HWj1 zHXfQsld#wP@Fy)iw=>3s`(m7Rs()?7EiVL>WT>s(%zZXY^>cc?pgT)a=ASaYo2%TT zDKpEg=J)q#wK=!fJm_QUI>pn-|2*UO#I;MoBR!pNIaQt7L%WN`k@q#t_I`^k?|rQ& z+t9`2NBBu4YMRZR)IC39}y$RM(_|MsMxCA5!rnN!q&~z8ktcqP^{%Zqn~A z#||VgSP$#O>0fD22CW4;9DmiQ7la#I)Geo|{yrhvcm5YRn@Ul0`U(<{a~R>={7^~E z0XfRVIZCZMr*zk3s7HF-IQ#MeA18VkcI}TI{z9LPwGE1nf8viEosv??R2H|5E{WO1 zZPlhopn%IWRi;EswNwv_rFn6oQ4J|9~z{Jx7pj-g&4sTnwEV+>&fS) zkBg+nZ7%!mUX|x8({h?P>cf4dv2-RxM8H3<=QqHIgb^8m6AnGqpQ={8qHE|VufOm7 zN3S)!=uLc_%38N^taS-L$;0xfU#acvTX`xg;AcX!I&&21PG_bznRU%-n*594BaIvG zp4rIHG$VDbEQB4=Z88^4ZCA>#>5Y(i&Z zjk*6ImPl{hWP}#Q{^TePnS!`A9DW>=7U3 zHWY*F6w%60y}6^93wqnwEG{q*b1`i#(p-8Pm8(R#p?Pl6%PK?4I)6oZR_Y}wWo7Zo zh4$d_bIWEQKliq&B<35r=G>3B2?6eFUGH-X+TI_i(fMkq$SAw0c;WxB#P!A3{SQlA zjcw__3Kc=f6^PJ%OJ_={L>s`{%MYkugI|rbTAUG^zSUX z4Nk%LdiVjjAlxl*A-D>-0odvqM}v;dpyN>(kxW&|=njuS46S8)zYvF+5#Ib#2GY5< zXl42$Z;a@kQaw$*Uk=o?`6bh!a+Kl$JrYeiL8qyJMPOZKUyf7b#f=mHUR(K@NvuC~Llce3Pg9ai+=4Hf7 zPJdB&YT9qlIj|3OEa&T_aa^6`zT?9;&W&JFn=+o&@0G5SW0C7!f-AeLFSEA}Pyrc#(4Sp7@x05`oPW-S}_F@wIJ z=P=KFJi=J7d@RS?u4K-ZXP~~HcN!8q!PxnwkylS2!Lq;U6|WF+msPpH($WIP0MyUe&{HO)s`23h?QVzpy@vHP-q z+Rbw>#Z$PoaVE@2^T$%>d@H*ko>TfER_!m5_U5?}0xWRm_+ukil*EMD+~fY3#q>0+ zKKOm?abWycOg*m|c!m@JJcA~eCi}P;eJmV_JrZ-EkKR&f*~D-;w3Jtb-#q8Ik(W`s z2|Z)z?G2!gQPi`a#w|AI_&>_FL~s7qxS8>%^fS!pUW^$@LZ%O?9nj|Ym{{|a80d`qdcxOyt_r=W6$C|^=@jv%kPC|~& zvqV`g;Pms+P0)|6aE^;x*pj$yxijuyUf>7qEoJJ~BY?Dw#zIQ;&!O=+;Nx|8LPb(y zj{kt~sl*%~|8W_lxvw2z>hvR?7ljF%TbUqygxU)v-`Y6GjhwI*G3JkQ2s8dN->!eVTkqW+{GFPYw{<$#a;P>G4Zmdk{mXCfvMqJ=U4u34V)e^oUFY$WP&Y9vz4jG6wCf(Y`hlwBn9?k zjOQe#HYk>QHc8^lb#Ga%sP#$3Trb3#?c#|&=e9(aB^0ON(zQJSbJVimbbu2NNaDdpXkwBxW2 z*?Op&1dB{sBf+Joqw3<0e>7HghGh4=9Cm$leN9%)!-` z9_ad+ey+ASW{0L%@siyna0z=71v_zQZA+eOym-C@deYC9>^t~rX6ytV>PvmpswsLx z3n;K?Ro-tgx!!Cs?RM8J?ud?}QMpQH)3J{>HJ|f2S88HuDrWNJn8}2bc#kP8%5_Cl zLn=<2rpK6r;30VFwCPu{wY0a2_S(WnzQWcr{33sNz(SRYY*M;|m(S9Jb9_Q`mV5(v@o0DuT>aS2=)nE}-c8hld&< z8+C-FIyn`(K`$(wlwRnDZy+oJZqPdv54#+}ep2dKnU55kgp_fbaLx0+&Qm|}IS~I4Y=XV!8#H0O8)>Q9 zPkh=b!bfe775EcNjov9eVsxhbs_zPvSe1`b*HCS67Ls2xRJC6~uSd4JGAw7xQI`jC zU-Unhtm~US$9yy8mQw;&F>tn^Q@x%Ebxa*yiyc1nPSv!Rwr3=+TMKsw@=z}vmUgB1 zU2UWSUAY+6CMmQoF*UA8iKEg&$gy^4_`NJceWCwnH(gaAb>xU!p=+d=XZv3Wo(oDH zpX&I}P@n9db-R=h14inBekrj(;MFNRLw&fv5oun%pBd_R`&$wEbI8bffB&pIq=cL- zcW?h@gx-~mwC(+~*Kb`fb*xUdv8n%s_2&?-O@`O?&)%?ggVb>_8D7=@!iIAQHzmV2 z_Rrq9b)(dgFBVDHcO}CPgpY`S)L!*S9giiUW5C%1 zz`D_5XJ2w8)Un6Iu5v-oXlq)YED`Nn396_7ReaRP zUGr;SEJ8F^(oN$}<86ydiOm=4f4h(B6tI49c3Zm=tS#Kp*7dy9acmN;lz>5SV>idU zS>g*&!x;~qt$Ovfu~c(2d(Z`o7N}2d*&p_;Q#EM4wj_l;979s9nGLBYtg5>Lndpg-pPja;8Ag(!? zZ&otjG{l`q=6fg!&qTz%lZ-o@?8PX=g_C8=l4Vm6cRHEx7s;|5;yzBsol2HH-|La1 z$+*2qSpJ5%&y#U~kqnss6LDwt9*fwAWBhx;R}w1+;rC=OcTFQk`0IyhFLk8Vl(C~N z<1lV3=ruT1Ul(Y5q}1^VxCaxc)-}Q0V21iq@0#_wm~rDgMoZ84p4Nm}r#0r1I&Kk- zxTg_UfH(&5SR7ZCC6QUp`-UtoU?Ly7p>3uSYxMc5l_I`#?O0tjt z-dkBMC7w%$w)K97P#|E`yrFl^?YXy09d{>NuR`n5(NZEcnSLw6!7)(kfa|SJl69`=8n7&5!QH>Spf`2T2%Gj}pKuLQM?$Itb@P+HR}HC`v&4_* zGS^8ISN229fvY6x5 zySqsX#G1;3xF5R~kKC@UVk_?6T3)^84z_YDo-5YxKEoE_&aKHW#+FuXSX#LeX$<#Q zE4Qq@jjl)I`vyE4SngXQ;*%8=`c@Vd-cU{fxdac`6^Lxv(uEA~46qHA#v8h!Et}Th zLN!~VU(j89=Z28zo}9d|I}KNY>^D|!*@#QY3q)M%-M9%iu-8|w+gPQ)(K;9PA`27g z#x{JMQ9&xAbrX8sgOh{LR%fU`?%_ojG)MjCzuzqnyg5uStcq)^wi&OzR&EnZ{THtO zqn?3UutO6a!T$4`yOkH2s%hz})GH-MN=rMAV8!+_TCHgxf%9Tm8PJU#l1rB*oNQU5 zIg=#K=EUcmH^5xA0h5ahnAN&|rZSCkK=$I9%amv2;&Zt`Z6Gt?X)*;?zeL*HEibjh z0(4o&soW&yMhWhCB6Y&?M9MAAiIj3jKuXNxkK^2pZ*`a!<2*Xk zloCjXJq&$k;af_KFf9ta(ZTS&2%(X@lo$(Z8Hq<-CMwThC*$LvYFBu&lzArZ<9YQP zyf2o@Q~ps%RrrB?JCTnK%O{{lPN2M_0Z9r`XYN3LA(wBx*EhJ{f*3C>LEbR}@)e}w zD_c0O2~nDh6v`8hC!8af1X2P*tAJjN3y%|@YnLF)GN%pcgjd1IT6aivYVb^4!U^U# zShl}7FkVC+k1YVmS#iK5dNo_>fZh`)6>%ndfj!zyNfkJIKp5whL&E++VJhiMV3+T* z=F&dgQ7X)`K_g1xlx(hD+1%LPyy*>16VFEmTA}lc!8XF7M*9WJZL-QSWd7O^S>D>z zcV87r-vT{BgVg_}6Z9L+0Ts=BT*Jlm95(&C8uO&U&h*R(#OCF|y5nR0>y!o!GAtRn zx%4#<`-cAg`G`r7hmbpPphbHfiED6Q(G^SQOt3%YRFBsiGvR)dQUBNg3p{)~LlsGVe7JV zj*Xc!9O0X_DNa$|U6kEOrR*#}Fga#7<4(d=+!=Od3}-&ArBd!tq#*&6If+W$L^T!IZl79>*usnKTjG8SB)=+iy!Bh z`$3E@_&DyDv0OfW++~gb78alUWCupt>4GIUSsp;P;dAZ5tYB93{T2@PHctKY-X0ta z?2gRK(#LolzEk!1akv4pT+-`iH)zWBd34IHDnS!={D4^fntqSd686B3O$m4cZt(rH zUDy?=Gh!@-;C#i{!Jj`AnY_a|4jXc1ryLdZX(~+(Der;hx2NDmsj5~)Oc={gFawNhRsCYER(&)lX8NesuPykNH`U#7kU`fXxHhXeU;LgW(f?(Ck`!j%S_# zwUw@_y%nJ7)?Uvbihcn4<_cNwOUaD?WRtilbYINC`vBahdjU6StS4&#=E3kq$t!=#e;JsPMy$6R){IBP)1{Kz}vx{&7Oa&Z^WGtK7<Snxf>xbg$G0P;+C#O=0vZNIK>-KTKwm63W_O$;87akyJquW?Q;H(hP&p}QU> z9e>~K%-fCIIMzw1OMD~IKkJkw8B1o~@6| z#rbZE=r`@uU-S~BkSp{%QbYwr6`%m{U^d0LEoF@H;z6sKerw8eaC>0KC!iNt6K%J* zJFCqdLG#h}x{@7je|etf;!%xFA=*KWvjmJ3&NiW?#(kF=7HFzda1$ym`%jwY{-NeM zXxjTHt;fyp{Zosre!za_<<_)}@exx!zn|8r+L#SzL?gn;EzwQaiS;7oS zno$AX{0EIq64Czu=_O9n85zWhbQ><*@NJv=i`{_q`nJ4Z7~IaT{!o{0Y9ZZ7JJ@6z zJCYc%-l`k)ew?TG8f>PiVhxR(1)SjGxY_h^vpLeTadW!PowUU}s%xokQxJ!>byji) zjK>^1fO%`excq0=v#lI^5VIh~y-dHQU~w#^U+K~jORvEytv=9Q7=$A(I z7WQ11jNy_o*YzC0oZS5iwL`p$+FGQ~wt==7+WJKo)x+hYwK`7S);$(yBpl;%2W$X| zi_;2@G9sk5gt*l@Cv<`~Xj=GEk5 ztjlzqQEPYC$7~|q_y_~G>~DKx>Na~ zc0lLb&(&fU+c0zUFo&~o4pwK_!%6CQ6oMij7I;RR1MPj(Np0oJPVL@_WV?YGrrVX~ z%RhuA9IxW2xrQ)9m;fYoJ?FpCRik4Az1C+KdYzP@6<8bOS_P?Al8I;J%QT*xx~Oy1XW(7KFhZQGOoxE_oUhx;8NGg^ zlX`J;cUKg?Yd^l<@51XK3~Pyt@J+ZTOxObY2wbp49D_N0IlNE2Y%6u0W22Xc%6i%S zH9Axom(a;Y`sN03qcK_d&3G=d$G z6Du)B{YEWcCK}*rn4BhiI-chi6ZAQx%X<%c}%UGX(#-)$Z4%2jn;z`@B=$=(#-wDXWxIS8F##&)|Lo-Cmn;lAlE7j2($l8=rf{ zKdtSdeyVGB)Nx5^$BES1Jw7wI3Wmr zC-~Jm5aDs!Mdox0?T(NoDAcp|)Xg55gN??^UBJ~mOGo}Sw2vVh3pUyG=v6ZE47{V@ zN{Kg5!%Go1bKGvVPbS(SPD_+QXYOOdh$*ER&L&;t>!XwV1(utp{I;3e>o7gRxowzLHg!e6vvIe!U$R9h-rZ*xzx}_I(ilKb@t(N8j zo4Nt@@vhz4z0w2Pv&B~RTt8pGTdVPv9>rb4;vZ^HIwx+N~dg58Ct>$V(0Zh<2pz$&nhHc zs=m`L)LX#!@g51JmjRj@=ocT>`}G5%-soTV`7y5z@G`D$AU~@qZM>jM(WN3jA3A7} zZV)oMZ|^Ap9|caK^&(p#lbyAXyelP|-s-e|`w!gp`BFx<4;EBJacoKv)+2rOA&N6m zX)47(%;Im5CCyczY85Qn$k)t~?X7~v$y0iIetB^4z{A#OJ|C^dIIr1T( z`BXb%`&4_?{e`!qsuJqbD^M8x6=cj$DHM^06?W`(ML#v-?N^eiCTp0A;#>Ylw z&G!3LCJRwnRc^pzN-Dd7axxcQv1(zJPiW7gzIzzr4l-EU;=U$y;+&UkxS5&?domvS zQzrdRTf&}e71;3??9*UP}d- zoZ~!%nJfGc_}cYXRSvyCyS*c@UHYAN3)|6_b(pL7CY<_BZE1t2QM#Mst0Vz{4TW<0vM(U@to(kld%EWAciMn#m0zaE(ij`VSu`<)a zxP_IdbFO(RZsIMmIQ#{WGZIzE`&Fm5FjOP*Ub@37H!w}t!#%*blhy9}z zv)~>xUuxCAeWhG(G4hKn7cVO8uH=$8;dl(60B44y@8J%12#omt4;n1;TRCa6sG6Ad z(SmU&S{3MbCt452t?2{n!Digddbfo$zpLd(?P0PY%av8UB;@P=jXZ6vXDk<;hI>9~ zds5`|{k;53yma&wy>>LeGjerf+Wt_%0(mpuc$xwJXvG6Qto^e}*p1<@c@Jyfyjk(s z5!$>NFDamH+a6Qh)Z{K4)`NNA$zeBc{h9KF=qD}aJY%%7^;c=G4Hq2z^op@pjJg7r zbF)jescedBvTPf7Fpi$96VngI)0qW#B~zFq>;}xc<+}j&mwjIOC3$jz4Hm$uk2xNJ zC7ApI;60dpq3JcOCKlYqbnwoC%z`Z_@m^mVAWuKQYQPCYrmLTiaPpu z>GYGJaO}fEGoxJl%z!V`5vH3^iaY@|KGP@R#@1AV zf#Jqzl!tX?V-jz#wTkIl#QOLQ;BK63E0_=5eXox{a8h%poYZEHKLP&)e4KKo;7)zY zUTqfqSt%L1HdeyGHe+ltiZs4OUb-XJD)Pm5xM$(*nLKBkGVw&q%)DKouzjE|A8c7| zO}yG?a<;LFyR_Fm^djSlmf2Y)F(+1mGtPb5hvH{DF6L{*o#b8Gho0486JDdi4WNm& zzBjgA5zeiKl(`*qd1_|hv*4n@i53%EAeU*v#68eOee}YDEBO>_OZgiQwLg zyf~O0$g3<4{3T#h%V1$~VgcSevf;fWE8aV@W4`080vZ`FKMi}rY5AmenU>lF2?-ot zlKJ3H-?*?pc(XRyIWE8lSKzig&Ow6@`W)dmf*m_tqq=Bh!XsV#&=z=WK zaKHBK5y%fmgm&lJM=r<^_v7nVkOv`OyZ~9Efq&wah^19Cm$apCV^iI7rt8|3g)PGN zH1)^5SgYs#s{N`R=ORq}4dnOVKwkg0rlkHv^T0op`si1z;Iw{!8~l{p-`My^Z6;r* z%^FWTaRvURlE-z16j@?X|Na`*W1MvIlU%w?n2R&9Kk6qn#u*)*)MS#TOP3Kz?|bIr zS5ijERjrT_YOU7yAthkl0UXSO8*E4WJvA0r8tfEs`%u@rh#@{~$gdvRkYKyd+h4}G z(aRh=uD~2;ZoF1CNRmyDeAUUL>b0nKuaR99)|Kg|u!MX1Ox@j2BX1*J*VHP!w?Sj= z_1SPfb<`(>G3p|uPW8@y?i<#2=eob??-o(}yBg@WM=IUTX77R$ZF+5$dTonE+my4| ze}RVT+{qdRy+)f}W1aT9?6<8#6SS*pak6H-nLsnT_6xjaMjAS{BTX*;<=~!Wd_~W6 zy^~~7b6=yLkp7-Qg9oV56VkTk7vS!A_kcdMHA43;>1{3Z4(Wn+D?7qnIXcXSX|~eT zl1|~uDckXa!2#(>l^os zx8qbwkWEKf5BQJ?r9E8uJ1@E{hwd4GMX!{oUrs}i*-|yrkIY~oFOMy}foTQ~qu|SGWtJ8U! zwrROI%0;cpW>9cp^Aga_=-f=9V!ccY-ikU`ojTmRceXP3!W3JvIzZi<2nyTu{m0#~ z?c7dEX-=C$|KI1Fv|QC)-d8`Q$+`Tl&+mDj-}8H(?}OFEZme%#G~k-YK+_SS6Q1g{ z5}ZmZc?)pctm@zhYj^IHZ)Bd{SVBLgC7wa^IEwbcJvopFY<3syA%vtWlT}qe={X$@ zZwvPMxnxcDw16snu(LJctlo@0pM&TgHt6g)A*#Vbrk}bqNLD9!e2#8`b7Jl3XZLRX zTL}L`#hwQOl)ir^={#E_ zIZX7?2-(NfE3{F&CsBeF@cM_6=ZrK$Ha-eD*J;I>8p^sRjB;9=5y8F5Hz{U0Qo9nq zI3et%OR=Y(Jq|rIEQI!8Gtg{kZ%m~=LmnIf>b1eM&~MnX>Xo@zjXSjos1!9 zSpS#)S^xWB2tF+iH(`k7=p%@hyG_AK!KZu|>UFb{?BI`sd4VrQ89Xrr^GwxhnchB% za1aP~>i7ZRurzB9^l#ZptSgLGsWWAIi_S3cSB2bN{%P<}LF&yfgRUO4xe&I;lFCGY z{C(&ucn^oAhzal!l@O~>UOGUi1iD@Tn0gzSVv+U}t#3eF`6$vdYj4|0=$Nl#?KPLZ z-z^A0LJ|<+rA8{a8*>6T4hG3^oTAXKG&WV6GL*dTHKO3=^!kH`EG$YVuD`zw(%NjW zS|I(8rtetQ^+0jF13KypxeY#w@+NAR)C+YP0r+@2mb$>{`Yx&-hf{}0Y1d&*wH!7<`dQ$d2Q=sGlo35HBM-iK=>M4k7knCG=39D zPt}5Va&Pcf;vM!=dS*w@E;vf{fqMk>wFzWwMXmqbI&Vas-a6YUW!X?x&QU7osY9Q2 zUp%d@i6`(t+i5L7(bvnO{ua^s7WI$tV==~GgohHza|tox23F^p+O&80)czTG6iS+Z zoj#@~)=K4D)F%n&#+%`zn0s1_xpewI=PB4x9EhuZ6*}MpKz`4T!8@TETn+sYiVB10 z1WJjkVa|k^rNq_J->!t+$?3jHT~v)b?wlZ37x~;hqDWlDSvc9fc;J(7<4)mj_^kYT zVC36{LA#Qptj1cTGN?8xadwcy_oSjnq^q>cD0Bq`wMzFzBOe8{sFPaHqVUmkee`ET zHUkT|PiFLZ`F79Xkk6=3U?rL&-lL^#miTlX(KBI8ej&>r{#dzHh*pb3&IgA8A+m zo&|<3c*`2>|0Emm9q0PSM~h$+FvG)?d2BeHh_HHiu5GU*Sw{YAJ1Zx)^K&h84(-#q zEl^sTpp7d*pe7__?x8&`5EBWX-BR^#w^#*#;ZG(oI^F&4km#KntChv(e-!-*Wn_yl zc+kg{MiwJ zYr#Zq0`o)8o3#l6*rkdZwZ=eB%-mbaW`?Wb$rOS8RA5TaJ_cVx;ARM>T&$`YCSGJg zOq?meO3sM_zrzFkj&hO)K`jCloi@n>snRYP93MazxFZY`=Dt;JO<8ud?68jURpNr5 zjoKCnvg=_51I_eHWpP4`Yt&kpBOy_b3aH$&=9^$?keqvU< z8Sc^!0#7yb+Ai(G;D%3J&Mxh*ySp^i^E%VbRKZ2TtvUa@2wkSH!}KN7jRcwY_jP0ToN#w&|`*Vn}l)Us;A!rC$ zV>?EwMc0&dTrzL$bW~%9+A&CyiWhtizLRO0<_Z+`2tuXif-Y}pyQ4J^CG&t=N&Iyr z`Qj^Qk~(wczt`r3z3lAxjaqlIR~d)2vHdI6jcCq)BePeUB7`Ag4peuv{tZ-KVu12lyL4?31`~pF!ho%jQ>#bG00?4 z(n<27T`^p7@OXE>5M*VJS@K&b9dDtO=h2+oZK|cs_y3?!&ipaJrqr ztUxr(NJtj(5wa-5o_yqB2zxPD%1y9}z;_(>W3t3mZb+rey<Jd9@9P?Yx{?MUfkOEiqm@J!?hVlKKypNB-dZZuq^oQJBKuonm zeJV9O{&v3U;R4lUT9Nk*Fdl(RY@#y-@gdmV-BQ#I>jZtKveGk z#ZY6v)Z4^f#reE4Iipu`T6?P|v)(G_f$k?CKhXWu$^%-&eL!nm+S%PYvlG8N_3O`M zf5YF|{Y-Y|kf?5`JopmgbP_l;9ct)5pr6HT#|hQm*l~@Rd1`UeDTLXqs_5pSXUH%7 zu(fvs(20MylJ)HijE=rSqi<;iQhEmNijRVNfFb<5>!dUVIaC3&U{DWH&RH0ZpPyvO z7h62-yhVI`ZB?ksq4o|5dXztnP?NI0GJ3qFhhK|5ra7hi@b57#3xBhu3`C+mrsd`~ zY_ksJ^1b4@iqxtTz=KDQ0C69Hy|1E<(!e#Kul#jxr-VBt3D?KyRReC@*6 z?6eNyL<46B2ebg_a}{Va02*GEYn*?M?mkZEe;Mk<*ztM;r*o%4U^tBOwaOia7Dt+d zqVGHHr95~xybP;w=7qKGHU%pCTG+DXWRt200x2d~5!?d%)53v9cw7l3wZI+D((t^; z*LMrQD<9W9CD7Syy=IHzPl5kiarRRIYXLmo>Ci!*uAz+A+y7Wqb7dwfTByy**CPv_A#?6z*f*>8Twel4_%w1486hFyTpLD3V}xbXMd#UnXu z;Wv+&S)vt$gdvm4&t`L_4ol?^ajuDRD9Z1#@Q zoZIni20NmlMXc9iyYFjLwC1b-!*_nP09J(`YetkLlYyL+bz8tO%zqZdMhWG@a#jH<7_ixPYM)%;R5=Mla(%W$a?sP zl~gX9O(n--?1>3Gw&2cvm-2J1b3_C6?ZN*#WcTwM2iu)5BQFVTnUKK_;Hw=&hAs3j zA67u%*7n3qjwAQqQi8&8~Mn^#Jps=jqDu7B<4ZXq}z=v z+JSn7XH!%|Sw#RIEphG=X@v>GN+C$Kr=nTy#$9S>D*4o4D*1gx3NSq$K*H0c%X7S& zkt>!tO0EnH;rvoODVB@5j99M9KG92YcqpIk{3N2RKSaI9>aAH1%xEpt*$3NR&n_0; zRs~;+Oj^S>_(xc6y?o{snnix)ik}5P4L*~%y4Pd^Hnn-_RN(6fdGFyKOI$(7q-5>* zVpP&Ef%?ymFYbv-kHg;NZI>w8z|D$=lFyG6!>gBJUvjd37sVMC^X!e*K})?lRe}TZ zd$Aue>g7d0)piH()%JQMFJb8Hh1bMh?bY19T4&Qq znmc&Ogjjb_C>e(n$*IvbJ6dnpuXX04B{NxgC>a1x-d#TqHe~$G#oXv?a8GU{N+E3- z*pf3n{?N{`w~FYkYV4KIY?}tE5qNLyJVCw+C$@W~XVGdJt@p8)1-^G*^*pjd0#iRM zm9IoUOgP7#5Ta2X9n)XycCr`hv177`{-68CMT=l_AWPT_@R%vlW`cvz?ul}WyiiXT zx4W=&!5r^&%dpI%)yI+xj-9TzN45HCQEkO+UgpDm!eeO~JeFo-H=$b;$N6`|W0I_< zW_8)ItyU9k0s_8Ya%`gN;AC|(t2M_S*1pF)K<@q?Ydh_&>XGAbT|9F9t@ANf-(y?T zZy8bq$ktX+A0j9Ocs`jN7l(m!UHGc+!j6w#+Ufh+d%*B5GKHHp9)AyhA(5KR@KLJi zX|z~-8^xNQyUgbXR|RJW=f{u}$O5~o`LZ5Sz}{)L5);~wYy({pChg^&z@!Cg<$X_R zk)P!O^|1_b1P$}A?%4}ZaB!*6t;m*x=a#Tp8AocrF(Jhp;EQEy(AHmQ$E>`rHC_L@ zcK@Q+wdNJ~AU-j-V*$>iOwcO695T%+iDCaH^8#2{OC@*14YsgF{pK;#zl`8($QHu1 zHkD8&0FEB#d2sfG^$F0f;1qu|vRK^*IZv_B(+Ianzq1T?oWW)K9qbEFwM=BiaGo5~ zR!j@kfOEn>TKBdBk`-wg6eCF({bLCWM zb{Nfz-`}&rG(C7#@Z8+W-gtNta^Mt2k7IgUPs0}fXHdTNv*Hb~Y|Q`UrFjO*I8K*@ z$C!GK*3`~DSy3}iPENN0O#%XVHW0ukV4OZY(rtFr6F#Hf3#UYwO!w4*)q+agWbVM(l&x~XFkdYy>ynMTj2?}TLfx1 zAt{6Nr!l)939Ruu1A-nQ1jO!DvR_MHbxdkE%R-!NAMkx^3jtxLv27EuMK^uoQ09LZ z<^M(i2J_=1qhof~8OF?pF*9S#EEu!=IN?x#{>TuW0Oj_SjDluaY>Vji+qRfCNn3o0<6bJV3Dbf_aD7hTD zO<3>iz~m&g+A_v`*(Ptt-Audk22K5#2t~hLiuP zuM*e48~qZgPU6jVA8EDC)BTfyoxC*{^?PzsEP~c%Fyvc;AFQ3w1!qNnhdGfd3+Qxk z=||npe5?y+PD)r$+R$cLYIKVq{9`Mz6OcNEz+z}joqZ$Fe-=L|DNFo{$^u$)&#iSO-Z-^q{fNC@Mn5a+L>T2UkNzYNPUP0YI2Hx7I~l8JPgLJO#s1^h_kTC~}E5zeZInWR5Cm(DXI@OsFeZ$0mQz1*2ukVErg z9A0mxIdiCAm!YmzlT((BmuaPUF5Y(WZG3J~jJw#8(p%Xkn|k>Wn+cBwH(GvaIbxVo z$hHO#7o57up{QII1V?5N`ZE ziBY`U+WR)m^Gb_4Y}tUn3#||AH~NXY33#H+%hkB^F|7e|8t42J0S<3 z!GXa?k-vtp=vc|GwMIv6)N&+aC6({+WCUrZEDy!H`ORj0n*Jz0v~Z2-q5IYd552vH z!b9+VXdIY2J&>_*Q+K=YBX+<_rA`8_%SdvrmQTAY zoRdUG^J|bM4ym`^w~FK(_^k=+1sOE-{2A+oiNVW)m-bkM)zOLYdAbG>u5`}*UHoOu zpk_MRg?|7m#2A%MNx_xxh~xWffgEGIy{sM9Y+;=e1dc0H??=o%tmp}DH6@%}>d5u8 zQlP~9rMaK%`<_mXpzaA~^$s7ELiT#{0w+qoGVoMDa8C(5+nb35j&y~xQPH|C@n3^D z;ZMA>TT6o0JPTt5eyqjIEPcuasGPQp5#rPTW4LBfkB*+|Fyh+;{&b^&`bJsEXoiC8d zZMtuj?CO$Efm;KFwmjY!0!DYl-uRQ#tKnW!51498=bNB~ilj&RO|!EO9?z7r(Pw|J@-PsyWp-AAWjv znjw-t^G}mLqmqJZo$QdIVVn&=)RE+$hAZ-g68;{O6Nk*+zs1VZS`V}xXgse%^XUx! zNw+E>YDA}dgXSr_4C#Iwp?UxQr42rL-@nr;!vc1^?|1tSoVcQ9nvX=PtC5kW8d~EA zSUt!}(|SdZ9a$fF(X=3>N!z7887BNHfEOq%B3sL)vod05rqmsWpc&S}yV@cnigX$5 zdhDtSvhIC{A?GrU#-x3G+N7x2`%)}J#yB+5-iY+TC`M#IbimG*F65%=!tSq2duMwq zcF9kjflB|;d@{FS*S0aOD^4W*X7{M3C_+;N!rT+}`I+RSEF&+*S&rygJIufq6_&rz zEy`O~k|ek5lH3|cPh>-PL>!hXXcow7(rN5l;9dRKE$zD87D4-$g5C?WQf6q0TWdw^ zjCgMsB$-&j&jBV)1$UvOzmo2!43lONdHMG*jU@j&W%imq4|JGiu6y=Jc5-I;oRi-< z{Tuy{>g3R&VmQ-&3cBQ?HRHa&2bpHspD@ZxR)Xp_j?g$~?W>rU6&1bzw|ZThj`P?E zuFZQZn8S})X6yOL`m=pJgYQjkkoA}>;L@-_e=mYIPKNzPw~;-j?P8{4jH3;$zDGtZ ztIz*s{Y2)`q308@U8!s~tUdx};45VPffYOf?`uL+;JF>JbYnMPe>Pd!cPC^kR!zaI zA?u*2H)1mNHnLrTtqB>ss;wRNAvS0ql2l|amrXxf)MIljL<~tyyW?zD{3?7HDz##n z^0;cTdwMcW5#&MOWz!Y2f>s=+R=|FC*o1nGI;khSkeg3vLH3->CPZO9D7ddD(|Un& zKR&;Vti}w{qpyc<%iE~E?sWNezthDnf6)IvIF|R$^Nr{)v^eZ6yKvWu}r(Vr|tkI`(&avac5&vuZ$+glSjn4H z>M*HCtq(5Go#&keKWD^+v{mNUhpq8GR0=JeSzYD1w}+3$)q6=X?V( zdWN2Wg{VO;`bnY{xB{C1}ckv2U%OH!0PdRMW@JM{BC)Rlwidtvsz*u`keySj%l=t4i9Rq$89KU!xv zn$=@Cet_&=1|6F=1NYZb=FiezA8*-|rSW_zk34UsW~ScNa2e=(i#7uu3iYOOiBgH^ zPiPJW?VEUW+wF*xXu1sf;-F=mXYCO(rYK8#G6bfsX#grtA6gEr%={?fSJ|2=SA_ry z*6AyG_0ZiwYO9||w7t0JUg*p^rb|KH(A}zlm{TOm6h$L@(21#5J+;8PO-T8S!=1Egj4)9i%I{9k_y>DMLI}wtrTDdj-#* zW4XZ=tOj5wpPkVH`*={P7NZ-3kNO4=3fLw4{QltCfv*QD12^GBAuq5uAq!k@6`haM zIz#SH{5=c3r9M*Ms8^;TdmmPwdsC&6lf&pGMgx&V_q>?VqEs{WeRI=5JCji;IC9&PuF9X7mK{rH83MF09TCM=4q|FUIxK7olcEEc=f6|OxU9%uVN0OHu zHOF6tkKI*>%we9xG3x0+%!V9k&ImXv>@Soha-q&Yi5_vi+1f%aAZp0%Sg6U(G|w7x zmpj_bk0XACFX-WRb0xISD=k~3F!xF!X$wDJvPjYI1f@;S_e#uIOBeTl`9-0HZwV?5 zz=|dmbBaHvb}_Y78>h$0IA7+y>p4zRJNF>t(zxiJ_D$Ny$|g|WpAe@jnU3K<1+?f+~sxr$;-=UL!QT^2`kK(RkvFtaVH?IEd6_VL)THMP^t{&QO zOo(NwijW=8Lv>YY&S#nK*6E?Qj-~J()uGRC;!C&(eJ6HpltAT(2;zu!AF)%_d(mDu$o6_P@_( ztwt^hsRMPJ*;&x53c-A$mrj3q=&mC-*h(w^7-e`KyP=wKe$g+K*rI&4BMK|3`tykF zf&{b+MZh`5H*Ob;OmU}vjTezb5EHEu_f3i}=vlxP4|Lp$xEkA6{oq$~Zw}Jl{T!52 z3z@MHWu6Vc6H%R!DvJ{(d!t@mdeEDAi#2PzoY-{=wBLqzi+WUuAj%0mn<>=_a4Jre z5o@ElV>05KX&Ld2xG;MvY7wG8Y*)^k6!)H&6K}A76)0VE3aBO5kyU%n%9yDB&rl#v zEr}1w=Mk2cs2&`mzEG=JkCl;3%ZVB_7XpvvxNOQfq26)ndhIyH;qYGtih%D=pbRgv zA;b@nyxOI)2uZ9?oTZV>`fYdV2|2%WNjlu^g6^_)?!CTzy`9vS2lck(#LrZMo}sj6 z!k#+Xyhm~Z&0 zVa~z=JivH~2{BsVjR{3Kl`1OspQpvgBi2N~xnzu&%*DN&4hfkCB=D$NamrA@k-u^& zV$FQnQis#+2c8_=U!q6~wF0A_omJ_K@;SAbb)2#kpw0?Wm62f{r(Q*76_|^3xQd<%nt?i^iD-K6ESS5Iq>#ca)p5j-A?tiX!7j5z$x_FUKR+@q=0?0) z>%|ZGpo3`7hQ3TXi1%tk2XQ-hWc5cv<3Q!qhLPkg$Sy@K64n1VNb}xl=KFD{fzICP z{;F4?SzHKyU5bbh`I>55^o={SwR`fn=rL*6R8vga!~PF7LU7!Q=%-Gg62F7f!cJ{# zC)T>NqCc{>6@PW>I?l=$4g@SHt;id$M}E?#FL650pXm*~-(95hH6oKRH>QkQu}V-*c(H#|FOj`v^)9 zCBiQ#B>chR81<59B_~Z~4B1n!N>h*%)-74h(y6||Qibns{kvkxcz=KQIq=Zqc)Ch| zT2`y{d;DT?AhP<8-T1i9GHN_3gn z51CpdnYCuRqJkvjNYJ ztUdkNTm@NL;1A2-M`|E)0*RO5J$lC_1*8rwIUhc?Wa}l`zg|KN3_Ra(SC2^{>=(pl zI48x3*3kxxHzR6x<9wWYQ=!x!FAaT3mwVg~iNk7w6KL`(L zV2e4K1-{K5y$9t%+;P?_oz`hFR;sAKjZA`1G*jP?peMYlPZB2eWWYb1RWt?fsh2t$ z5S^j`@s%>c(W$*jos(jPQ$KD&`Q`1$qf}_b+Kv&XZq2~awLr9Gomwahdlw+;!d`^v zInW;MbPD(DYitr$ah@*IEZ}K^dew-0L!?H6Ps0B0NmzrHsHoGJh}@}N-IL&rSf{n7 z$EX!!qz!_OIo&H_G(wa~o!0IowASgw2(X%#99QyUtl9yLZ%J!d@O&p~N!go+l8&R+A&Yt! z`JZ49Pfz?kM)$9a)(%9L5>+sqXI>>p>XgA1@zI*tY?Pxk1UJ-g=;4-zx|vbJFX2Fx z8*g>S5&s9;sUZAE7OC;H6Gad<>mK}_*^^SUw8tvE)m?bG5`}+c z<@%oe>o&qidKRPvD;6Mr;rTyPox*P~+fi}*9xLxiT;Zrxg62x+dCJ_rnD_cfjoc$z z5g{cl?=iEty7Q_{(U-2L{daTCb>L1)X|XIAW>nI|pY0j7{9HU@R0(%%|*iB62t~SH-oX z-xm%%?Ie07$s9xUoPR8(8)=q2|kGXw8A)cLpjr~fBgEq8@_(Sk8aqf zt%y`=w$6nEQ5_i>GCgPEz}6GKKQkc{XsHVax^hqhoy`|c2`wCG%UU?FgPy`X3GhE) zC}DdR{T>&3^DuW!2_f_A!0t(Tfg1QR;95~ZpjPGLIe~=( zznDb7-$Gdp*P8p4wY#o0t!i9#<6&zSS)41bg??0ddXE+K82#7lmLS21hIGD!*nE8~ zQQs{zp40+4JIsUGu=w-ydBCY>g*WM(!4^M%2lxa|!A;p1Z*C7D3n2UssXUS;_qCBn z6G~u5lhck?^bG?EI|nphfs!k@El*2s21jpA&jS1cV)P8%XF*RpqUi80eJAa4;XouU zRW2F$MS7;WXB;$uHcN$33w)c9j|7-Vti&5&vD_V8&d~Rs6g|gmHZ^z1z~|$VBjLz& z06m2@Uo!AFy)-A#i|&#^O@()~FYWuIE|U0k^+_+9E5 z&?gO9lRj_&*cju?eRhzYb!Y$j7wte-d;O(*tM}})A#=tKJ(I!}FIE$tczPu&Kyn+m zA6$+o0C=fOz_;%8Awt?~Rhy5oLa)x*SplB7kQJu6QfXArF4o9k$@CFlk~$t5Cv|KV zXg~N4xS~1Q4y?i5*b(2a$;y*B<#e@s+9@{0stU(=p*e1FN?tgUJSR0;y9gS0?i~H4 zgynYQsgb1n6@ykcfL1*^*XRr)2hsQ$9i!R{_KH}$hHPX(0W{JZ;rRS8LKdW%8xXfl z^vrdPlh(Bfuk?+qX1D_BtzK zdpsR>IXATHaHCT!5b-P5f3L>buESmO=q~}kMC?eq1d;5p?(z;3&XxL^JQo<@-lh(j zbY1)Se4 z408WH?Z4Ohfw8Qzx*m(!5rwpb%w!4LKm2lNquQBJECP12pt2%U;$kM0$mo*==si`M zivi_z;0|DLgyeU$cWA}V^ysmdXEzWc0A@3UYc8o?3Z#-BkFdEw`e^VYlNn@`8)MDo zc@>(N@hfCro%%6{4odnhQx`(_9E~0={{!i!#`TMltHT2hc|FY))m9kQ0>yF|rzV@3 z#}z`}Q4TAEkGKKux)=8>$;t}-v-&ZlSdpMUodTUO+Od6zs9r-`kde1ZOWp~|Mx01e zU(t~*pg}K0o?-ZklN1-uQqVSNB}?#Iwx>Abck&Xly8t)LiT5O~9tN_4?5u@P^lpi` z+YOP2wLKCC-3B8$Gl4w`yWq)DE(WgaWMCMq1hTKB?jq0k=7-@AsbDpdU*zaI!H8IK z4kw)<@{D{%^k{dqO!taPCi67Hw{IV$_Vc1yaci$6A~uJ4w#ojZ$)9UxoXCpmBF(&` z36}Z!`aB$uQhNe4+-aTO@d`Y3x%WTtN$*Q09p ze|bz)bCHWjvMm^BLl2Q9qR%-QJ+&n;6k2+rIWee^1tKH)3Dxw3;BTdf=sLa0qsBJKkoB{xB7Emp=iL--3ZnX z!}@WOEeDoh?6q}kdEf7`ZYAs}4`%IxJ!f&hqOS#<4j1HhOsHbL$Z-IVuTQc-P5nQt zrZz1H$FD$+J*stLRT41@{Z3q$&3ZIG@wJ~p4xR&Dq!HIENb<^igBd}Je_H_D!Om_o zkQ;hZTX!6n*tNREHsmx(c-qGiM{C7yir$A0PTvX+>4kfrRpRTZ@D7siqi>L8hlEG{ z$4gZD?nk?Bq4EBVc0HBy>{vL^jBzvS9UV79C9CLjSk63tG72{{O&Xdkkp=oyZ z@hsXqN$Q73Y43y;sl2Trm@>^}^t}OaRt;nwVDG-Fe|vB+IXFxxG+1XMNf2--EBGF1 zTq62YEC-@8dAa~;7aY)vxjU8rtvpJaKAtW5pxcq&M-ZObGF_XeTnx~$-yn=W+ zw9~ZT2O)Muh)!i_DwY+9cu;p%B0h(nIwd;COf97dX5TT2%(*+iWGQIbdr=k9C3c(& z`E$0y+k~*u=K=5PTN?I5qGrLbQQt*H`Rz;igiUAbLcbbwBk8y-<5bF54BlMbSLv;(vH^Yn9pA3L8un{M2ztY;2pN*U#Lu;P7Wcf5Q|=As z*4}D!uD=@jNw~(Z`1`@_(|8%RZ>qL^itK7wSJfwHbq!ROb!rx8hS!DKt~hpnu2Kc$ z*Q&Crz3Uua{am_&tK+B_HH~f7Gxy@|&*%l6dwLX<``pn6y%*?#A3ZU9pl?$zq+Xld zoUn29#7{lhi=LE+SurA(3;21|lYNK zsgEV>+@2T*Tgwkd?6Ra;5h$hiy@sm~(n-s|F@wcN|F>sQQZE=WX0WnGoppQ$ z!$;3NgQe69M4K9Pn1d+C-5-~vVZOzW{l0@zGKS3q9t^S`wHfft{e0C>tn3wxs;uQre`niiQ#lR z!k_D{;AVe?>gAHgr~Lh32F)7uG2{SLUp$6H(S33SbMNZAiF>Qct2(uDPTo>p<*j0g zY(zI&SeDn~e>P=5&xVZYpo(+(mxwM7(0Z{x+goGJ_t&Ui>*95PJxEV1_1ViLb;B_` zEWIB1fn!ZC=ITb>FV;Y>JHDnBtZ9RXf5u3Tt%m8t62?$p5!~vo_sg0vRT)?gO^&E{ z(J_o{&A0?`WoO!rwNdkY$||6xa~Ai!N24G-k9=7czfbiFSPvyM3b5ao&B7SW##pXc zl7W~(D^8AW>5+6FLlmLDBAjOAN1tFv-g@+2@B8U%;?sk4-&hrdQ7d6@MXSQhg87EL z;;N{R!RhowqwtT>(-SG8B;HI^b^0jmIXX&XteEJFsH?*Ne9A6hC4a#<5MOu!IqpuR zsnL=6;n9DDrqJg!P2F|$RGQ)|M=5HePEk4k0!3YU^kjtn+4jw!xd(eq*kT4qai+~47e9F>m&yYq(WB#r_ z8V%xA<>A(pt-z3+XC4iMSJ7yEZ+KhPiDO~=2ghRNFs)02zgg3(^73IjI1+OMB``7G zoRJ_5)n`+-^Q152P4tApo4zz`P}1nwmH(S#7ajb+qon_r`d2AwS89}!j!V2zN*Wpb zY)WdUsO&RHJ))(5j7!xbnlml+q*0CpT|Q$>@>xe$*7^S3V{(u3xqp`S^0$MMzP7Fe zHC>iE2x@A=$_nZF8&;OSCQn^iR()kj2`jrTpQ!MH6mhpUNK3kA$PI4_sM?8aD|6TD z@@|3!vU38Y7;*Y@v0@=Z4PO5o^q+BQ{YRH`Xin&mRyaCZQqdO56?!->S^w~R+^mrhOWW-WsB_0RK>DBdziBC)F+@X>%Mu=gA zd~Bx83(E98of6K0oQLnne$sKor+cf!DgG+eD>9AG|NBAPxfS`dfGZ%WPRv>8Qs4?0 z?Ze(BdwZ2nu3T0FC_`QQKeeU0hWrJ(KU04)0&T}`XgfA{o!;7FwD#bjCybWTen4c! zVtN>{P`+mXyQr{JH@sCgQga}r^=W^z;HSpbImR?vnb^X!iR&=^r47&Q6hU`k| z{RweuXpa`+(Pe(pdA^7<6Lk@1zDF}SbH*3wJeD+PcCfWNPe3*=Qv-?sJ`UVMOg9k({w*JKl%PiV>3k7lFyx?^gf5F zKRs&5PON`;LEGtE)$V~uMs?p6#G`64Ga0-3su^ad_6q+Skb>3t^(Lgx8<3>Wr6hsu zxFolXT@PgmHjnLk{&vumuw%BC`lrFR+dg9MwVQl>i*Vq7h5uj!Y#Vi5@Z^01y5CPw z5_sU;(Q-q+}M#BJKblo(-rldZic?o?VkqP<+C7Jtl0nC)tNJSD(Ja9NVkuc#lDn&Kp|Ewz>%fz!8mRH@>hlS-Ze3W} z3y*rM?5>jK+)KW)@HPCGYHUS~OP8Kp-1-8YG>GIEGaX9XnVh%WP(omd!vb-mZaV&fQuj_y_n- zHf9#qtW)*6QSLDT++!BX_^Q5pm(siU3A$!yRV$ByCZW}Nuuqt3wj}cOeZoIu!?3EZ zlWZ7FEqqn3y&NYki{<%M^|ahc#$&x3doJ>=&P;P;_C@PGjb6mmmr}gBwuT!m}Y_Mz4j(%Z<|0k&ofzFUWYjytZAo~$7TYyMD2hMn3J8JE< z2^i&K=Plr@&Nm0DSTyi_DuWwV0b}JnkCkIN!A2y2b58Qpj{X9S@sme+i)D;MPjrs1 z>vyy)aA=#(p{IdEPq&m{ooCd+7DDTsH0U$x6MsFyS_0mVR(=b%hZRe-8MMkPxK+It zLBAJaXO~exdf?}4(e#9&MO)PxSP716(F^|g@6XW3L!UiE&i|De`gHi;pP{#h|F380 zaqPwyoiIb2hyS%1`VMyA$7kraKOE=35zA-kOKm6VOSwbXE9h&+;ieTMf*UOyHiN4W zUSr)KSV!F@i>s>T>aQ-ZUTROT(;CByORJQuxO8@@M@fC5F8;B;n`N*qk{nk0E`$Z% zj`d6D7)3z$9!OqzjPl1(Hc_i;#a!Ag0vJ34 z92prFK9?hp=?E=@>-~z}e=8y{X#~O;fxRgY>2S2yTSWXKN^v}`*y&|}Yb?3~nw+Bm z>sit{$7h@p{PiGjAb3`i^@#ns40&Zxb`9wYD6s~2BpLhgaVKzY z8_rZ(Q`Q25bD!BRWN?4t+-tG>zS^ai>8282#lFUF^;FeXHQa0`+iPOHJ%sWc=kCDX zMAvK3?hj~B2IM`zCD-p)y_T=8`}E-U^zH;w06`a4wJLRX9Q@LL_crdCgLcbQ3*rG$ z3-V0Z9H@ug4?Fj9K4x>CPrIP=d{g*E#Ip?R_S3L#PtmR0m#_Q#!HOvk(AYP1?PNk~ ztKPy)`|0`#0|k(V!c(XfyGqpRdo~5zxo$iDVol}G*i{BDr4bm_RsIrZ9m{<3vg+mX z*!}|g%0IYXBIGM@%6i#0G`e24Q`w8EtLsi!FUQy9KU&9E$Lbh2wk8`;$9R2BZXB!Q z@`{Dkm)6muZav;(@6H%v* zd}J@%-SHQHCbBT`gy?`rQ}2_GzxcZnve`W(CQOJaE2(@Izr+jK?5!OIsQ>brl<(bM zQr_X6`C5W!mQ=x7F!j$dOWX@@8}8sCLRw>vfWK|0zW>V+h!YjIu3^D=fX=)3cMI@^ z;=h&hgt(NqGALCEiD{61O}_EJ1(iARPd~rMTQ0NOHcaod23NwD%?WSLMyK%fG+jql z4=uathSl;7t8cz}CE5zj1ds-?-9}?aE*T*+JOGM1@!+KzYBO1mlys#A!eG?@h1CO=xZUrpn!V%W%Ud_ zbd=2DIFRolQ;v9N1~N|bs(U}_Ba)@y5?7`P?Uku*IiUKoO}}Am;vgQEQa7~q|i6w4Af7BRp5#$FeK$_bA3oGZy(*jN7kEC^;JUA#P%8eT^1eD>@R>?qAcir>tCTw0 zcKU9A&uWgb+et!^w6w(~%mqUo9Cc9iP#dxrkc?_TUe=}y;JHww$Dquz_j2onw>SED z#~rs{-P3i;l^R!yDuFo7#g03J7gZ`HpT^w4YXbtKu%t&Y6#^@!2x#sj<&8a(mG@XJ zpa3iEPg53S*6Ta`JsFm-1UB?YmI;ag|F-W3#U4&xi@X}YzK6J`5g^uGh3IereZ>qu z2ui@5XwlzlRq$BHFT?~%JOY1c8k0TUx!i{N{vGBX=u>d~Z9(L@Ga;*mkXM-<{SdQNpv(gDIpWnYuhm4$+*9eotm7{{ zA6~k?xyB3s^!m9Kh$Q5utSH9o?7M)ibFFfLGDU&+kn4l)T=Q8;J8*7TVt z|88UU&FcCcp_fOa8mfchSX4ua;!a03L=+&d*NHgy!mbp4d?9^Zj`}m(!h5Vt{m%$| zqo>2?npOtoA1ENa$SJ`G^Wbx>=fuhmKjw2#M2iI;KK^sRbS})eU{nc{)TUpkBGRcVwHKNIXJODq9a5`zEmAp0H0tPkxz90 zuIo3h@~@es`>XJxw^mtJ`&YlU`mR;gYbNT~sGKvE0nZsipP%_YViDm%Gy1(5o;3RR zxk2)$p(o~GY-(HC%;ROgwaU7>Q(N8)e85O2?v2)!8$>_N@NB2PR&(#xCGZxTlp9z! z9px06lyig6PJsuz?n_>gV4R%=y!G$ocVzmDku{Pz-@&iSye0XRvw*)x{At#==1Y-0 zjF{ZyE2sGUyU7QWyNEXpCvV;PRYcK&rwQ`#|9t|hLB{0icOw?T;4jFp zTE;^maMiKj>3pkF$E}clblvddY6#s#KKgW}8rko{TqaL#8VCHtJXLTHKw7tkknxvs zZxcS5eCCYH@ajUW9eE-yRz-^0L(JXwcKDNeJ)u$G!=gJ)bH*Wtu`HcIT@H&bi@Fin ziyF23RxN2u84-0K$H<>cRBt_MWCGS}*E4WY3(UY865zRr2wwHE5!6ub2@ysBjgr)1 zP4`ca2ap>=20yQZR}ooKPmGch)#ONPxU#PamPBxk0Qo*~*ScTgNtp+RlDm!)haO1Q zY{%$S;Hs;s36eHKF=P@VrblDDfDCmWgH&b37z5pRlx@Z@Sx9k<7jcK>G0Ir?f#!kl zwJ3?m3YM@D(p>1T6y3F zdn7rWqBYKCi&{S9fd?-0l>F14wiFNe=~47ql#0+_J;u@joEjN1mcn=|kiU*H=4!-m zLd!fy<6og@$=;X~vT-=+9HyD0c{5tHEk)xmjE(;o?fRj`s>fI@{Ii+vPf7U_NC&_v z7g``69ZG&W%w>b0A4x9kFeKAg`yxQcs9PkuK-af{s>}6=GH@QuWCa%nC*;tas2HUytrrKpO{1;u`)4fBJHb@R%qq*)&!ey_<(^cL+d z5}cLrK8N2nr}ZcQ3~{yQ@alG3BFplD2C>^-7B(SAELWo=&U5)i{jn`r1+0+Te-FOF zA09hSp=63C2I{rK&}GQ_ESWvQ+;(`NxlJn5JYV8p830cBe==HC^xob0{uk=kdB`WD zXGvj2=-m~dNm{|=53Tco(Vp5hP5|Cl!3(O93sErz&5ML6Z3@2ky*yb zvBfFF3NuljEzW|rV6DRV63wWBiZyGkhFnECd+HW zC~@pYA!TIzc1rcN0phRZ-;Oqod3{?`9FdNOxAa8VxfQFD$XM@^Yu)CAEYh2v6?bVa z)~c6Ml52+kXHoNql|GMt#Z&*L<`HHX-&cHctI}7hf1guNwY{m8UIQwzj(`7Aj!*q?PeIxz~kInV-qRCqRV#5QHRxH*GIOeAs`MG{2WOlK~h$|z= zCTvmvpn3e5%W|4MrcJ+z(W>-tJr`?VdvC`JL}fcJiZUu|Q*+VE@t*suNg6&?E(#V?97Fy4CyzjlxP z+ATez`aAm2-q`$ojvw{Y{3 zjDTem?|s`wMk?TRL3dO4X|%qi0rP?PD+Zcl~Sk^`PW8p7`G zS@7_8a+*a;3*M#oJ>^vupf|#I%fjA{Whwaef@E&lHKjKG%njGHZ`N2RqDD>d9yTrR zakw9^H}yISt_oteqivNxLeI1F5Sx^6W_BST@dKY&nZ0)}>(aFHF63e9>UN(E?UdTj zWv^wExtV-&=V?x4elX9Q92Xz97LL-3xMh=`h1DMA=0K;S93 zLaD=U<;ziwBH}GFfMjWrkWW^jvaZxMOd35%c4r33PPy}IX7$_=*j3LN@%T5w zt1_8gz`l84Cl*2 zur*O9+5=1U48U~WYer-TX&+oIe60;9Pz-3(DmZy#K)#Nt1;~%U)WAqfyLBw%%<;Gm z^io0p5Qnh`v(nJS%NintY=V7*w{2$!?c9KPVIY7n1){~ZK&CLVnSC9)88|9DlfUkY_r5vAOjT>bkneP$g+d~2U9jmAAgu6_qJ2R=^wRrBP@TGn{clM@v@h>P>gixC~Q z43#4W*&2h@O{OR`muhuUW-u@GLT@NM-Q+rwtkvi)c;gc8e*GKt)HR1^tfi+s@QSa6 zrtAPZiP5fWUKJCyhzc_@-*L5M_+ezFH&DuJ1w7NhfQ7%px&DS--8?H6Xjsi}%xahX z{mH*~)M<4ye8H>$4=&KhnMPjpM{X!b1_Y-51v%Y}5&z+<&YePdO}kJsGs;V5$3TmE zzCzGSX?DuAYtg*IIIa`QO69aQ>$b~ZE<%KAgGMwJ3tst^~z!g-b0t;;BtqeSjQ zwpZ>lsr|1B_)NUU@cB3WzY2vNmOj25tE?6`La?m14nXNlK=uUr7`UZPp_=Lo&M z1goS$<{&qdiEsANYW!5z?XGLauA%IheI8R#j$i%2uWEQ)50AC*={0*b=>` zF7~ZdQcYo9A*1Ks;r~2jPuyY=iti-+*d@H!7>z}iTMxBpW_D73-u}osd)*^IL=^qa zXgfs}eVEmVZxA6McFixAP^!VKTA^PAH;X67f0PY{aqegoagjdJYw{`a`Tt z%A%qre|`pxdpc1*oV4mb>J4p~ILH1lm;2Yz{?y}`wHu?7``-9)dQ6Cx<;f{i2IwM9 zg>FSCW$MNV$LHOV73~aK?UeY~8#*OTP6>F!pe$BslmWdoQE0~8LAR}-SPr&H`=0Dn z+JK_737G;S)=B&8fiPxLKN`u3Z=(4u)#o}*kA*U=48doLN*IOyWbEi9RkB|leFQXO zaO%78?Q&?Lqm*|U^L{4fJpzg&db>E9Zl_MqlT(~#znLQA=S8iQ zp_&BB4m_bha%WboDYqg>Xf_prY4ME2EoPyxSw{`#h3Gx{y2#w5Qvu3kZCVw$oWxu4 z7i;%hu$l~-Jxfs z!Hg8W9Q<~>=Qi&7Wo&=@M9n;dmoAP9rI*6qa}W_eCU!|&C@ny1N~6Cxy){QlL#3yc z+>Cs;PSt{F!XLayC1ULLk|b~OY0pBJbDp7Xef@Q48=`+YtrU|88w>XG-e5)Gt!{SR z%0m@>`}GnMx;%A5LBZn+8`O8F3x*goV=a z7!%A^>7D&~FQPV`hZ8>sRRKr4B9KX~9J{)>KeEJE?^6ZASI6}`uw;Yl+x2Vqy5wJ= zF_b{YmLT@L#Ld=u2vcA^Ij zd=XDD-Tq!doI&(s-m!OTJ&&XWgKk6?Ac|(IuoC^TF!Zm9MUZMbcP@Fj<8tJ%B%Unl zeV>Q*eB>xU*EHv^uw~L4B#%kk_}?o1%cqr2y3vVaAtnm3e;4Co;ugm7xriZd7xsw2 zcq3D58}dzCeub=1kRFl()HYa0h(o#H&!{jn^zdl61v_zjG;5nsl7lg}r*XSRlKx?P zTqwa@mRO=6w-c5WFX8@aznvX-$`hfXATp|8Bzf*I?f}6}(Gj9%K#qP4Ta`FgO3AP+ z(-Tr9;e#=tP>{8R$J{&GcFF~nY5co4UYejh1ZmK=ldD$FdbqQ#m(Q>Q*~SCv2C5gU z2mn{~Qo|ZQlzf1Ae8s&$Gj7J-U>wHeMAS(QFbW0~i#WRhraK_OU&N#XPNx+rUg;I>S7-MPLgt4@__@wXdr1KPe0Fku%?g>we$f^cgB66v*l~@fXSVjAj`v+w|ovB%F6#WCxSj@$AqA`j> z+7?8XEuGVvuOsw_mjh^nJP+})_kP3E&J*PVqHnL_<*C@XfC{bacos4>twr4x6#b?m zlprps)ADY_KY@-N@CY5EdO-!UE3(CtUaA0A<|d1#FIK zNEou?jFQE6y^?VxCzRJNyUePbni5C5^i`A-zQ0{clv~}vI9*1qx&WvUSUh8oGq~vcqro zvm&vWWY@ejmMZm{GDb-qD)M&?@L6H>mT)y(=&>Usj0|M*LZE|`n5Or|8g}x z2a(q}lzm$}{LTE&#YH;R^P#AH-WD@y@IPhX$Q67)Gftli^%E3)a5BU-Tc-c zb9ZZL-*)>bg(720WbTC>qhJxj2JoYZS;mAeV?u~Je0D+2yNIN3H<#ESnr z{tN9wVZT*d!Cq<14G1I%fFyO<9;=w(MR>z~79T>Z<7a9=o*x7>eg(jBO2y=I|<7 zW^*~L%A)hfYrZ~xo^qa&)hkE}-h0gF{Oxb1jE^JJ8F;}w^e8fXLp)>Fw_?VXp=AEx z>iBPR@vN2l`H4pTY>Y4&vh<#=?pfa;8UAiJ5UQK${{29l8>JlcB;tW9kG|Dy;h{L2 za|x`6ofy5Y@`ttUxpZ1pJzS2J_9a-|wqt~idopyvD^uLhk!i{-vl6L8@)xGtH<)Zs zO~-f-CeIqogPzb?ZoK)bhfa51L{`?(`JvJ)M+6VabP3##_QNtm7A1L3JLn{rvhXzG zuh}yvDtd~exv!9J*9J-JMTYg|IAu7w7srOF7qHLZG^0Ox9c)cZy+-r2>pNzQ0D2BB z7SuXu(@)juNscF%=sIJYgzOpVQwu#KIr0Qmmn}qw23nW2&VV-%++!7+vtw6zrbZV6 zc~hoc=7}1FQ~asRKLJra>r?L>5?_$?VIykw@HgUtz_-)8I}ZB_x9a$d(tAS;yO9?->?j$&po8ys0aR}S8G z82<~I*AvWYLZL~p-Mzpw7s==USoU$J5Zn0m$t@ zc>-|^J76z#z!q$O;s3YyF5poV>E3vCPtPUEkW5095CTj>AOmtq02P$go=GxXmEocU zm$N%4>Y%a)ui+fdVFIk8qB{t>3Anq_&F+H6HPLt$bU6eUFRQzIX1J`Z;@Q#TcOVJ7 zPD02ux8(c1J>589oOAxq{PR8ep0DyeZ>qYgrn~xA-Ce(W>#et>;K!{t_)CT7b*vcy z7HYeX7D>+nG*sU;wC4ygR7Ym`Di>c-Im3 z;sm=Z{hte%JNt6j-0l-YZx~)h{pD4Vv3shn!$`JASg1%D0hg$Vc^}bOFtWT_gs)-C z$Sp2t*KDvC@-@0B=TvCX=a|P7U+kDg>=HXWva1;UCsFUnwpzo`eq-F_p%%1@ZrLY5 z>wTa#+diMX7vHcA=6pNn#vs1 zwcdi;ug*izIF^mZx6-^s8IAsRt&VCwhO}5h6$Y^`wwhQr=oA4m+%$tiOxMQ z6u%a%Tb2KVvd$_MO;&|3nV1Qk0krM#e9InXJ8;D!+G#_|*reVs`7HZ3B>og86v(2s zro6Z7N3UI1^^-~GGtvAo-7)ml@HBZdh=*Z`=>wX(E5{gM;I$;Llkf1JaI>&~BTt zd(cS#Px0-i_`!9q@Y^e(l6278kUncO4J*Dfqmc0gq3t|ZWJ+!Ba zysWS5dgj+vutlhhE6#$Fe+R-x9981a;f z(Dy=H4SlbvUCNxbR&rowXfG`1jDo|6ccC|W3O-JIqY}+v;$8mrIr>srF8NXwM9c;B zrJ7tNOrBhcXTtU2-IspBO!$IT?qwabgj64Vt;52oQhxk+B~~0kFO9j?x+xgl-auay zda6j9>x9=WXF9%9;v%KcGYG%=;VL%46#Sl2=$%l7S)bA6ir7N-1XSOX&~g88mE>Oa z#x08FexI+n)Da}s^q!Kh_sWZICk)49#uojuy0-~5!v4J3=Cs3Lir`J>9sXf*jte~y zW+*L~q0C`+?C-(|%SyR3nvcB%XzZ!K2`8G$I_u!OYOO);_Rg5q{a-(r-^E<0F@0f1 z;aW4`4KO{`n^z@_hgE^%{ex3kb`q;aPWxA%3ek=SXalT5tE2Lx>32rh>M`-5m8lg4 zkxEx8-u|BG;j_f0Y}cg0I~sirhA3pcLY7UP(d@H$ZowJufPPGf(6d5MN$RrVDhA7~ zN?D3{_e99w=+kN+A-y z{q+w2I!o~N4yOM1-(M6G^T(5}G9OmKVIkj9SlVezH__KNJ-r>H_0#L_9A8@KYlc6{ z6~~5E5pNU6rbFj;W$?@%Ab4@7r@`lprG-j-$!pO zTNBe5UJQsLK3>xZTM*-KNxI5y-{Jq=4Szb=g` zjYx)JKD;OCrP#q}L7M^I7hyA;eE@a2(`WY$mo2iR^i0qVj#s(Y)NY0UtNh3_PW17r zCSemeWU*mZXPrqDvCqjCIy1y%4Tf@3RtZ}|6>LkjxVl2zVruv2&-NL%K)0K3*h0JI z89K6!#x0907A%#vWMf_ev-Uw_ZDE)H(Ip5H)KvZkEonOT{wcHbJB&2x^wX%|8 z!~NT9Jf@nQCSwgORcm&7Eb8b)%zS3<7fW-uyb{4Q>mCsWb#*xlh6zn3xLXp|G_)^}WUG9Vl1^SuH ztY`$4QuWghF>x~+L-pF|)EV-f6Hmf>qbK06x z(h5vG6*OFB2}!5NKS(iVxD;2_gT5 z?ybtpc46`c{HJRgm1&#hh@VZ{*h1C;@N9i0_FOf(F-wSAPYHR~Tr!^-k;PQcxbUn_ z_CU<9IIaaNfUT3oNc-8l))e8b5L1k!W9yx+QhT-QtGsKlA^~=2qkBiMhVMARyN249 zWDHRHT3~!MkR7aQNGFEk<7@?OJSs%aenVb3V?4k#2y-x!KCUG}(R2*RX<7?pVsL ziV1H#eo1e<;Dpt%$_+Ne(5bA0PJlF!WdiAes9Yd(^I>mHV)s3Gi zj=*-T7Te_B0DXh)s7JGQCC`NG&%O=M%U1^9?hr8RMSD$*3}uHY%v93`e!AHUVo*A4fR9`>(5oHY^a{m<8L~0%qG|M~^fl zIL)`)l@aXl{eIm$KH`cN>`k*9Jt6!TcmKy~i(I_X7&g0|zI5+uuiJaAHyVAdm+EcH zumtVc!=QL0ok!X?%iR&Z4;~JWGC~Ngz)UJW^k2ZPN$4R@GY=3r?f1}lUjnb%t{7!D zn&wvLxMR=!D;)xUFJ>&oGs-SLqqWeGp)4~*>MqdPpHw7|9QoW&j0T(zc9)_ZS4@Rg zJ^XLTFU81)Ri<{}t38^Hf^D_cM0S~W2D?yZpIRl2&p0ZM{_6=TGJTLC)ygm?%d@sE z$cLwLqZl@fW?^acUKjk0nefGXUVw~gXNze!kXB8R%o0Aj_3z(lq-&!L?QpL zC(Mzd_ugx`ZAmKhBo9zMdcKD_Pc zH?=PuX{DKfA&XK-J1-V3QS!YH=jDckl3L6IWI+R-OY;CzpabtjxxpxdUO2xvw{%h7 z{cWQ7?Ae#pz054wmC6+EEY*)`P6V|cCAK>=Ss^Q`y%UyUVN<>vy+^FJCe#-GsKhPz zXLhmsg|504utE{w*MHRDJB3Li_>A-XC~yXhr8^|!9K~dRU@48Kd!tvMH%__|JI$LT zhAUF7>64iqxoqcB<~Se1yr10^rg_EeCY+sZyxXMaQha`-(z*X`SS$HOybUqC%30_O zeuMF1%nnP&GnKRNLiCtY5&Krm-xOx!!@<~@j$tNcF!q}gnr!eZU=woGFHR~}=NRmz z>fTYEunr09bBOs$RP3ekD!$vJE=0wi#mdf%{2%3m1>`@D^TL_!cC#H!9D_}cLs}DA~I=301 z*(yZ8Zl?7Y1_QqL=~9P8b020yyD-Yw4m%`T{ncSfu?{b8irIb*4wbuCyCv||**NQP z_j30{Z;Fq6jGK<_KV>ebRdDYse$RBTaLwzk|D+wD=*tSdnjd`*X*mx-mq_!n4s|XL zcG#>Kdz-z=WauiY5k3cPg|G#7*|02m721d%M%pFH$q4SoJ7>m!0ao+q`_vJ$B@Dm9 zHhOOi7;`p~b^~_Xc%BOFzB-J@Z?7g_@8UK}w<#9YDkKknkaxkfc2Fo*=8%6IecMWe z=`8XGKf~|Ngg!t|ZZYB+OC1Xj`Eul6VNIl8Jtg7QO+KLzwhGuMvcxw>wN=2(Exw$7 z{Ezx3dEfL-)1IcPb)~TGr2C^5VEC$3J=)>(t=XdF+ilw2I@J<_TEY}0wi>esHJtV@ z5+fO+S^NZ^@`I{2YN6t=Z*L;qlyEv#Wd&K-BLlvSvtV6*I!#R(>Z4MbWk4uje1WhW ztX?S}wjO?%v{6lJJ*2lHt`3zyIzZ!shP}Hk5#^ z#=MdYtmpaG60i~}EMZmfhKxCL_&5bSnSTKq35r24{nXi5TRe%j*u_39-`7?0n&WD+ z^At?$y1w(;*yA$R@31;^7%ObxGn0P5K|w!M_fq-&LFilYoQN3#Wn{}GPZ@OS%#J5O z{f1>KmST1l)-!ilh|Q+cj3s#k?!CI$N54){pDp^&GF0meFglEARCWgI3-j}e)QE4t zB#s|?=xzA$9}}WqrK-Q8|LU(!k6(jxr{H(p(}toa>FipHoITUjB@`x9hRj1lZ@k3Tp?_ zX*~REdTWGf^p3>qY^~5(tK*;Xsloa^Cg5%AC^TEbCbXUj_h1N{&`KuUM{Fe!%{3JI z4Zw$Cbq}-b^`U2K%!&vpSa*^yRKD9Zee*k~WLUMy5jz;3P$*mo%dRW0g@!5n^lW}0T7Gi~F?tzzAUXm^jnz5p!Zj1mO1BoEI9kWHhVYyrOl z&vl|ElYIGZ@&f$(b033;e0-6WypJO`{(T(RC@(X5(>5VY$4b+DT&oQloRsj&H4nia zAQQ9vL$R~QVKD{BjKy=NvOw7Se*Z0SW%c=&XfFmec6c(`l8uUphD9EGB%48354JIF zV=t_R=qfF5rEA z11l9?X0%2D|JBjjeQHap&pvjUGGM4gc1<**nCPSaQ=f&V>oPDaL&SP!(rO?n_%0bS z%=CngQ|d*A9{Bxnql0y>@u&LK6>)>@hU@0e?X=kGn=YuDzI%etDvbpT&37FpjkZv^ z6um-x{%xUWl`?ylx!~|EltU|p)ZNGMv}eo<%|^{_>WoF2_`3@1s&1EkIi)h}L}>*P zW2u*&ZAyrCixDb6z&A4X6hdqN^;wH)AK?6;|4sG0q+?6Bi0SB0F@MTt`z)noEiO@w z7&Fg;p3#6_j;=EocA6g$uOgm^l{FU38dScyUK_LJG#M}&b-1qxFGqbP#;&4kV>MPg zG@?onvk9`{9fYZ(zry552~ugMj)qLIUi#6Q8=97{8HEu{fm=j-RmwtG#E#ps-nka5 zlLxuY=%ZnMHVM{eldStNGN>eGgs}of=olS%82HXm=q&Y-gb*n`U)ni?u}R`KstfB_ zoO(r2oOA`wF&yvE3>DMG#aZE57~PG-dm2^LzONuQ=XCQI9&ia)r>vIq6pt0<3TK`W z`V{&jO2rw7k5I0KwHpgyO+HHOs9<8p;xt3YGSSdck&f$=b_ISRjH8fdMVfS#$VLlW zuWb@1(cPql9>IL16dvQg1Dxf1ZiD-3?=bAkV)#_J`e|4Dq57~m2~25{HTau7xaZYY zs*!+mV)ef#6?dO$z)F(EsX$->+alC104KdE&k(v6J*Y66ZGqj&Y?Td<9U6Y`l(|%d z->eOWMcD6pOxHS7O8W-G4&QgH+-(NpW4O}$dtgW5y&`P#4h~=G%?#`Hw0@(WCEgLf ze+9QX^qEldw5tuf_Z(oKVe|loIEkJT4|cCj2bag5)sr%C?qMMdEWIq?zc%#5F{H4f7|O!-QO{zY?M28|EtIF74q;{hZ3|{5 zJcDtXEhOx~S=H}eZrMR)n>`CQwAOuc1m9t-2t*ks*5d1{wQf)JJ$1$26OCv`vp6=* zZ6RHlxg>~vamU_<|HP{v$N1;~_Mn;peYB`D(Gs7bT^z;6n6{d?40V?((!GTx($>$E zys&W#J3V3bXRXqMG&A@(gu)Gwf9sIbyF#W~p<}U$p$!+o*5-g=x_f??|5i4db)|Aw=zr!#72mZzT%i5(u3c{}>^ z%p&!?IL~NvfRP>LD0sPkZmG|-1r|K_8aoOFX^XiQD|5oa>3w%dwQt}oeh7utkXSv9 zwdNYNrg);r#;Vxo&SR1Nezs*;*s=pP7uXnlN7HfseD?eT*v(6$%vO1Jm;!%n zHO%;9t1$6w}de#EB*7M@7t}h!*OxNk4TJX~81(7l=<>=N3o7`;NaS zdZcIhnqqu!uTsCb5ob}DX6?x;`Y*w|h$TXoE5nLXw_&5RfR$s`f8B+m)nqY2U+ReS zbo9?}i{w+T0PjVh7_$~Td(<4V!NqL-NcXeQ4H?}sw3qk}K(A@;n3sn0r$JjztE#}6 z%pHYD5mKNh>3Ap&J0sJM@I&8LZD{3;ypT8n-^vO2@~gitG{JU^N}4csgiZ+SoNZ5- zQw~ofE)Y4=Qw*&l_E6YJzbE~QQ1}XB(0+Jn4bgS*FY&d83|GC#Ij*lF4GG3A7oZ)H zcR<6lsfo_PCV)>eI>)Dq)oTFr84K1k=3KEm65WPzJ^c#o%giRw7bFEbT-w7qop_r1 zv`Hn{+k3bH-?*ddF~gjGs>h*fozNBzr)q!omO?CQT5t!}7~pIsoXezLKVOC=xO)9o z(W69L&x+uFz0u#DgBGZ0SQvi6$^N6r1OLA8R z5t;2?5MxN+yi@FLtlnwUS-R7a=SA4>3KL#Y?+e$n)AvcY4zD{-Wd@z6^yw93Uq0RQ zylMqMA9~i!dLndbje8GVOw_sobdjKkD1HO161 zUMc|JT}s-ANCjqum*Oe=_ulSkT91UdsbD7U_-1F3If?+*bDMh;^$j(>(RBD~TZ(lE z>M`=KQyjjrzU!7|L1&n^uM?q|VF;zhvgY+BN{cW z1$O*|X6S28K4~rPKlQT$Tj_{>@ZbY}jCom6x3#xaFyO3OKRBUjwYoYbFap>k&^c+g zg4mZOD+tfYIRBUsSXkjjbS=OG=ONMxggcB>?`ryIu%7qCZD(QpSeCFtiJrqERESTqBgi1F@1+LNzvn=2CX zAQyJT(COp7uC{g@W!8?vq6PfoD6=4D2&Z=(MxP!Og`y14QywZM$_TFeua>#TCdZpfx=>jS&D~>S$JOY2H%dR+%>|u!69+;VsqwU7x+ILD{egC z6NBWb;9*!vk~S$4{roM1Z-Xl{Xxm$XS+pXYZ8Fv%(43(Dj_Dfg{4jQ1D!!vUHF&`H z<+@Lxk9|p)d`{|)HNB(*O)S}X=;{U_##v?CL~ z1+n!Z81sk48ncgK4P4oVD)V)Rt4wmo&#Lub29>(HTrhSP`RD3Cwp!o*V{7W$;%Hc{ zdn=#=pRDP8YoK9}U{l!XuTU(|Gc;ydphdvScI+9}n1vB=WN#0R&Ja__W-PL>2Xj_< zp9HY)yt8vYr*Dsb{y(-_a2?az`9az@F*9h~u?LzMG{bt_0j%!4bQh$vK>uth4woF8 zd?nova%lhBuXAiA*2xxdJ#avb3MmJsAt-I!VfNX4ra+2s`KhPvrT}zvvY&a*z8HzK zvSvZ_%FIqy8s^}3MX)`IRhig{(QYICS-!NLYAm;j3~7Q*t*H_Jg^X-`pCp!Y5n;id zQ=DC5qI#(43+@xeUzViM4u3cg_QTzdosn!`uj{qEjh;oKeg(afg^eB^Vb0jhEj*yB zD_>;H_koqVt9{NUVKXZryDMQM^*%k(O}%8}^JORl+5pqbsjs94roozRuDTOyHgxKZ zRfH)$DHgT1a^yRNs#-JR_S#dS)|N^GHAba_#jvhMdkWsdFP?|xf^i4!f_hJ=xC(3t z?;F01SVba&2|cfPCy4Fuz-CRf2b7%-3*Il;p1La)eKW9nyDR!2b_CY32r5DJkuuwl zaAw#*_7XjO6VWo3RTe>tpxK*(TEOMT0K!~#$ zjw$2m9t`pt(#{E^T9jr_i?Tc1qBIs0{|3fyLoZVB7&r!L1z6-9ybUzsMen$W?oouU z1p9nrJwbQ$;a=>50W0a3dkF!$`|r0sSvlwko_dd=ke++;5k-AC*&MQhp6DNY=#J!k z6)okkZJOz-BZTuH72_fJPdc6-SK|!qRP<(rN@KTeO*iZJor;zzI`&d=8ggR#Tgtgs z(c9;6u|Hq%V#1~4RzID6qI&i#w6pu@?6QQpn58PKqWbALvyPxZv$+oS%))4i*tvZp z#=BH@ih3{lRbpl5u`IQIVxMK{ZKJhDxL?@R6p&uZD{Q9qe*$K&f?x^fqpw~tfF<-q z8+&)dlf-RmjVD}E;|#UNLc$ssX*G_oZ)~2a@A?X(MeW{-<8ZjNzSJ7(w^Xe&p6QM6 zs_WQhV^_h2urvKC+S)q{H7_}@sNSCl>pZpQ^vC7>a2 z>$O^i|{VlulHsClQ`Wl zb`|Dkgov1)JB6M_TA3y~?g-7ob9WsW*fVGUf%N)SCN+JDmfjQndG7)>UEP(_F*h{r zqE>=Mlyh_s^wfwMIqF^Isb8#g^%qP%)8r7_O4qm^D9k%^)x~dDUNLDTV&917X|O2+ zD+2$pd)iV2gSrc+oMOIMI*8phiGetpNW&rEJ;9!gsAP#d0+NCvL7F%NKfnGb*s6nGHqx#Z>-V&pD5o!xlTuzu~?+ zj1~d@)68zY|%2iAtoN;wyud*-pW6eeNK%Tox=@PgKN zxtCS%CL*lNXm0$%9sxYgCT4}>?*S|${~3>7bK)E+VL{YuO{=nQ#jL8tz%CkZh4ph$ zXmOiaC=9;3=TeQ|ppdXV%NNu7nU7qqulof0;7=}Uhd;#8hd7F9)0yLJ5a(pk&=pN$ z%~-cFEYvc4Pjv4Eu?BWJ)Nb>C`FYDdrJc>hFs;rt;D`1U(dFIX6<{$efEQ!M5$uA5 zdz=wj5N@xWsm*|kcHsTVx&JZvtsu^+&WcQavHb&ciTk6Wi}SWBQ;RM8a^2fq zQYgij>#la43JLDnzDi%JPxPkxJl@|iXL$C2U-7BV@kJg?0wJS>V zfYq@c{^H73krk-d7uD;FgEx~Feeg!r|Dcmr?pdIZPQlt*btc9F253>sTh#IvxBFJb zw(d82eeHGa2aHui^VMtioV(HY8(j5htAuZNsV^5bTm;`qIg=K6Q@yZo!qX%9sZNz| zlHL>ZCbX7zJnDw4Vb+9^-V<{sOu&D2r71?gM$B`JMPEX?(;;DxVjEfQ8Weo7{Qx8D z+<-EbRlB6WkOrVP`jcK3J|6nId~rB_PiY1%9sITs`UdrG47N8HDLENJ`#V-^$BOhC zS30czYHX~q9xP*H&NgML{G!WnkojtGS1fFBSNQP!hyS4X*zdp>;8(7g)&m2vaX<{a zR^tj}J8V_2VO6hT^}l&c$!55Q!GLSf?`!U|!V3EjN_M#7E{i)8cjlMRN8f*|@=c=~ z(YUMmTGq)7fw_KM@bMiII&Rfk|ybqGdqJqwUI zA1ywIcFTaJbg*(SJ;Q`%!+%BI1fG-D<1CTAY!s{|ta9h>8-CutpLDyhK6IuZ&RMKj z=ULwvQw84_?WV%{ZKg55xPRNe?Uwz-`jTv>*3W3k$Yp$8g6-{}LLqQ$b$ z1W)KvYO&9<&sN}q&q-HXsyNs;+->y@_Pv7Dn2oN9u$v3EN?FTuAI3gYxrgA#470zw zHxh6K+oqWUcBdr1QVeVQ6CpY~+6Vh}zk~gEVm?gSq?k|YX@!()*_TtuB9byok!ZE` z<2G3G=D-^e6N|yYMDe$38C$-yO-z?2XO>!hqsaH5-RgS*v)8DNke&NryC4n(X^O;$Q`TK z@7zG5$g*#KB*z*{VTFs8bQ@V*n?v*9JC$Zs2YH43o8fzb^wTrJXERD~@LlcB_ub(8 zH8A^8&b?0hZ3}d{gaPJp?WXX-Rw;W8YAp%Up)B8R<=vt0Vk9!yy#O;IPOryn@WLM4 zwh7-LTt(^yi?w@WdIS|6u#lEa+yz;k5L+1qPrSe}&3yg?&)|!ai7e zlnK)|xK{qfzEV)vuU($A>@TKQ4O{n@3!x9(^RZHn{nMUyORB>?ZQmzlue4iCSZN`= z1^>kkw2qYedU?&hTQPzaB5z%ItKB3Tf;)$Tr9|Gi059eYGuiMji zBC{2t)()*9`E9T`O3drh-0lFqp9NV}!Z_+LZI~Og7hs)D zn=`Mpvj)8&6Bqfg<`mvpGkuGEKgW{>q_eUAabe5r_qZ*v8W#&+T$$qg+&d3eW_CBN z8;6BZFNLx7&!kti$(1uBZ&o*EtJZ)^n~u;M4ByN$d^6wUlXGoor8<{zkQH7TKGMp> zCApTe68EoMj(tVA3s^Q^_+l8mM)*30VqA9q2)yLw;};frrx(5>VDCv!DhMyi$X{dUzNx0rn`RVJ}r`m>R zXX^e|HWhEXjYilQxvRdQ)VCU%Yg$)Dcnzug-6?ayTdo=V{^Rw3b6s%=&#$Q4X$bkY z3at8@JljVlaTe$pW);s?*}mYtEsV1G7Bge)JIqE~+p9;x_GISFw)KXYZI78z$6IrK z-}QZrJCdrh&zWw;*m>wC9VeC*v;>HWL&%#~T~>8up1JJFtI3-q9S&l~L7c;h|A zsBg&}{kxr?O1;!$Shol}BbptKF+Za{rZ1=c!*o=#Ra9OADxban3zsDh6?%D0eJ-gd}Xd*o5p8u@ga|_1sgu?&y-dUBeNxM(x&xyaMMYyNju!WGx$9h}L z6;sRf)h^$>KiUNi>v*#=n}V%%LgmcrRa=_;Z(7J-mC91B@Y^kl-&4fmq3&f0_DE9F zi=To8rGIY_o+m12;+}+-*Kr>UaX;EhK3ZLQ4*D&BUT<`_x=-TcjT_vIG#DvQx>0sP zz>U&lu{z?pBT5a{AHFc-2v$5EQR?r*+Gnh{6|cmM$J!zAx^z|0czV{I=15esOdb!N z9(x#fPIG>vsU-Fyx?t-~KCh0P}JsXa0R&+|ibhvN12EDQ&C~XX}kF=*3zUAumkj z9R z#}<8vD}K4iwUWXU3;ioMc~Zh!-8@t)OIaCuvAt)>nic7z<<1rh;Rd0#H5OXA=JS#_ zDZF0{w%&#^=;ASs==$enR1Rf`%VB#TXxR&_qOsjMhB+2u4aN5q3I44M3Q0GAyYE-N z_gV{!zF7I5Qe!W;=RM^G`|bE&XMYL*>+N6Q|4w_sz3*Yf^u@hJ_r2FTzUcP*euZ#i z(HHj>t$MGuxajs(zd|^<=!;cFKX|WoYEi-c?~iX3G~9fLImot8>RnSk&E4|XUD6h)XUI$(F? z7!JQr{{p*b=4Umf-NqDWxA`A27Yd(-)v52Icg+nyuZVuJrnK|)8fW*gxIByBiafzD zAn$)6Z&F+ygXJR2SO}}PS!u9}gwErGnRAs{R;)c&V`Ye0)R>HzEk8JOZfiRF#vCTGUt`Vh7Y%4q`zymW|k9HD*IBtj4T}eW}JQh@HSSgsC6l z%}VR=q}7$b780&COoENuYxu@M(+x{MpNM|Il<&MxyzkvsOMXz1GJ^`sf44Oi@iN3^ z;oVj%G^BQLGR6eot~;uIt#>$Aj$N^T<;oS#d&GO*ZOzhtpPgTQ->y6N-`9GluxjjG z&Q&Y#+P`YoUC#T(``>NN%in+hdW0iz-td(eC)mg%MNsjX(Qfcc@vgFw>7G&OX?HAN zx%_6VjIoH$?iCEylHIe?o!#>+&TbD{ZgHx!yCTimy^LlV@w7AeQkZ7ma<|aFAKI|r zEgtMM!t04)!M!W5Z<^uux);-vbbZVZh=@Jc=~=R36n99ek?CnNwpYU}9v%|pLwF)( z=s^!D3c?hGIZjzgMOc9_4PjNuh6Rq5)vosswZKyD_=5S}{=3Cds0aJ^?%qd5H}m=E zZFhRq@Pspll66G-u>SEyKo68Th@Wr(o?imgCQDma1jw5w%&R7TCY zkIIaqG7nLiQFNY4wM-S2QFGR!&Ks#*_Ab=Y;*6m|XFI8u+=M*EPUqN6zfoh`sJvRM zfLf-8%BZz^fy$_{db)!wy1T!5#(H8}r>^Lo(04PDbY%2YZK}vDgAQgUtmVFuhS|_> zq!FjRse6-z@JzH+Y%oI6`T( z0>E>B0sLkLOUOD!)n#EfhUq?AQ*{>;rhA*PMwhrW{dO9js!I8YqKp(=7Bv-hA{!N8 zg!k0T?Dk5*9^=F0as0vas2k11U=_6BNQJdiKs^$CsAVkOmj&G{!&?{tuVwUJ6a&YnNd&KN6; zi{Et1&5IV#Enm1088=;wAwF-BCzi^vKMd+5QJtoJqfWo?2^(WIInYz5#rE~s5u-Z2 z)f1Ltu{V3_B&yROtxm5}om{sp(y!rJSYA3?JBq6k)2pOchw9Q|DBG8Tvp>~ShnNkq z$9lY4th&dp#n$$Og;=>qd+G$bc7}HC`*H1CZZ0c#&&Ivu#%JGr>!QWZ#pQ$z;+|4O zZk}K2q2pU_cFvnU^XBq09F)zz)wT4t@`dve@9VAW>kaGcP5iniV|Z69-yDQBA#$e` zPc&NDXM@%Rvxo3$Pj5v~*=P%5rMJF&hvY5yvbdc)$QGA0W-KCNeRJ$gEwO$Q{bTG` zXF~KVL)@?AP8nZxhQ5?)fcnH8Ro`3P^yI4BnzArbs8dG@St7;Iz<{@MSAKtP4 zj!Nh<@V!S5Yk+kb*_#HiMrd0LjioTQETuOwt7@xj&_UK-Z72U7a@z)hwQs({8XJ}G*6bUl13?` zM$$-1wKUz05hJ`{87&iX4_V;hL&&F5T24jZBj|75)3C6iy*eMO)Sb>YKfcj^Hr9Qm zZ>G;4vyhIYD}o~>lmkCMyDP}ok6=HbwD=Av2dd~>Nu!5qc&}nrTgWH;Wbg^0hDKo* zXQ#t(TkTmT@6-XjQ`_v=eE~5S{13D!+iW>l<+{zV zldv%it$3m-r>xZXU=u7N0yKA+62uuSf!C31sdkzFf?g~2O00{-thqXGXE-NxmX28O zGXA9^XMUG|PUE?bh$&oyI77TMQe7_og*mF-lVOj*DpS1`uEFh0dc_C6`lm|BB3cT0=VPZ@H3t_XV|*9%K#8^eq}-k4}+@7JCT zg+0`_J=Tamato}_z}=Yep>MEzv+qOSc2z4Ul+N*GyVHDgd@rfThNuhPY~B zY-hB-d>7_uz8~*#lmCV#alQ*pn(hwuP~X{MK7(41O|XIsSfK;hih5X&VMbt-`z9OlG%^y;~Vjm7$1R?OdQwDjkHJ|K3 z1BRV{kKTR8P~*%S2`$CQU~rFAyT4Vu^Ii0hbxl8Cm0H#M`midgeSMaA`fs=I5KG&t zv3A1m6i$0f;7{p#VH$jIUJoB$MV7EwWP%Qk`a0OS{Fp@y(2J1|1p1D@h$}k2P&OTOT27c!6xKqomjCihgl+=YpL>E!SYa`K&pzu{8X8;A~w7bHJl& z^lzbYwO7;X#xY}?=C?f|!hR`o{h2Gm^i|IaeyUX1zJK<~NJ`ZcB6?po8`kgO_@}+a zuMQhxpxi zU(FvwOYt#u7nag9u;H;WSWcwg-)2d}&WKjb)4oCs4w5Pr3xR?Nb3f7~PK3Sc#4PN< zBN;vJG>ovajj34wj#5S`Ec`@Y8^wI%$)3*+`;az^Q9>^STQ_NJ>4+d}OG}s9tW2B; z>+^}0;IH@KH~8io1Y&=)yo17+d?u{tb zgq(68RzUm8ea|8OW$q*GW=T5O)NU0XXipc~4#A>g7wWVy)XE-&w@Q;4_cpO~XwVL9 z#EGyn$2~ny_d|Ld)3x3Qy=mb&idc#I(`sw;^maod}v zxm#T+2RGzw$}xm(iZI*&Svca%LEBy#b8I`z2CV3{{61@>r9~Qd_Mjo0iM=#wKh0Q~ zgRcW`MYwAAh;s+w{j{JPXB_NK#rR<;G}0056=K`VSVpsPuI)|3E@=+zQzD-zw%`7P zg4q@pTIpvqUKd5|C;9=du%C@P3-5B|MO6q3`?E%}ap!PM`vun)GGQC;@~dd|S(7`%y#mhS+Cr$I@ELZXA&XVhCtpP$JnMqT zkguZmpEZWRivEXo^vPKX^LDJjG50HYsWu#?c~`-0hLytTx#Y9@t7zkyucFtSvlU{6 z=lIl5Q?dT|D_BAsb`(#}y=BD+4|Z_u46VpBY(LPEX~-_i+%AkWU?sh!bP)UnSW2ZJ zJex!^(=r1O9RGQHnsL>LLE-O~OP>K1(bys2oiThu@n0a$#+?iTiCUllBv|>H;|zb< zGJ`vqVv4dzDP!Rq(FXIMYutf5$2ybI8DETcJc^z2irmT^A>X(SGi#++D#DHS>xHw1 zhz&L%#Z}{=F}^=C$$&kVJBr1WaaSHSZZC_75?PfR+Fhcp?2$ZU;H%?H@kKK}%f=Z# z@wm#m*kWUVNDWBSkf0C0xW|zllEM!;Y{7rTDw{!JoIg7i|NYql{>#}9RISI{jda?d z{YW1D`YuOWaPYAT$D_fNW6bezu-wrdbUV6&Gab31pM0JZzQb{KaD`({@J`q|J@f!VO9TT_(8|`;KTTJmE)n{ZSEnpftjoF9{&8mDbXX+sz;O~ zFL4sWLEaSjL%REOI&K?+(dNSL;}jp{?>s)JGIAU{sttP@$ISjTthU!cTYC9D?~Q?lwR#RtEF{gozZvQb?p{2twA2L zokg~tg?_)?Vl;iCc%~%U^auw zI+%4iY*8i@h~WaGbmXdnr@EUs6t?BlRoRv_98rv( z2-s*o3rkJ)I94-poKdY6Y~WD8l*{U4439(|q;W#X8tEn5BTIqVaikKSDA0w!`i6y)Y@0lrBtR6qDU{z2-z)fW>YT48fAKGFerRY-I?Qypgm{X{~q1eyU}f^S-(uIkw=IDu@b*$k&swp+GWYbOjoKG9#;#y z{8L$2C|Xi3e)cQaK+rf6Gk*dL7;Btma;`BDHkb}myA0WG#MktVGsZFo#v<&z&E4`3 zv1e^v-pumuPs0Z5k*~}B4CSP7v0Z;2S?#P5SbaP8^f54=8nN~^jMr!9iCbc4!1qgd z87pBrj@|h=V<`@S6_-q&GWF_fuAMf0`R#YCxbrUhGJKupbd^GVMQK=@gDcUy#2o9o zPQE&{(E)u>#Rh3^-ddweTn|4cv|j+Zz$8Ta=c4!%t+02d?)ms@Li6a;N6&; zf*<7k%t^whnHYgr<>C7+n0BS_?2UeQ{6U8cTscn&(4B|@yIDFc z)OgB;Ps!4jz8dqHSTm?&zPYi3%^w}S%AOt&YiL~{Jm!8PyTrf*q?qYFtND9$Vo!02 z7%+nYU_MURi7!CsG4$AuuR?1^q}Ny~(WVhQG#%y4tcZiMcD4*-9cD{*4d$Af1obHu zYJ~Ri>e!PikGT9<{RhQZMx}F_^rtXvD2#hXZ$f@vSi%*}pJGL&u*qfkjMd1?hSUn- zA+_}E24+e%)Pu=+@Un%sNC-%wi5H(~{i*%zm@z!$;+Jeyo_;r7$MpB%mH(Q6NA)U* z9RiMuUCG$!5#+&2k`7~~p4)D3#8-R8h%-mj=a<$oYGnfM4-CSQV$9Uri%|NAG3>-Q z^3u|nJ(zte`q$k|Qf}qj>V~nlVE+jDtftjI404{Ekm3HbAYPoLw^$1Dya?Oa7=;aO|PVxs$^m4~=Orn=Ni8x7@CVDvvE$Ns?gV;2Q&*O29 z07rl$z!BgGa0EC490861M}Q+R5D`!ZJ0(o=W1Mv?7vmqumzwDH?ZPn$9S0Fg!(w7Z zNTQd^L!3k}Hy&}4sha5J%5Y4gms^ZD$udp!auik~jigGG_&grx2yg^A0vrL307rl$ zz!BgGa0EC40}%nNc4mHzvsy6DDsy1`1DU6ZUf*RnCZXeL2=CM6A%rA)xs8aE=;gK{ zPV#~#dbtK1lj!Asi#W*vP4sdU9z_~SSd;iX9_I*f1ULd50geDifFr;W;0SO8I06F^ z0f{O#8SArFGpXTfjDH}_n&|aCfMXImu0wcOlVb=;^l~Q=C(+BDN1Q~uCMJ5hY#fv5 z>&aun`F z8cCxj@p(MX5#R`L1ULd50geDifFr;W;0SO81|k9`s?-!F`7zE~>%sU3vQZPgzT0q2 zLdTT|>osXWNTQefE#f44xj!IIa#$0++?O~e(aUuqPI6uoy&Q$M1@M6Z$Q zY=k6wxkAKA^m0=XCz+v%UTz+aN%V5d5GProiC&JvDx{J4HHpvTagG2-fFr;W;0SO8 zI0762jsQo1BQRhQn2PmT4?vRm+jK7zNdlV0pQ;{dBn_H0BP7ww$a;i_9DxCf0PTN} z=f4b`F-< zgGutAhroXz0ZrmBSC2H322Gj~lIUe*J;FndzyL*nf8Ud97>*yuFA)O&ua6HBVZ-yV zUbDYIAQo~pxX(jL@}GYO|A7QFiLY)w(nuOKX+}t*myz`d4>KaO7_ z1UUamgbvT^H~xbM&rkg4;UxLbW8goKfF|)RR*y8222Gj~lIUe*J;FndzyL*n_ut7i z49Ab-mk0sQe-fd?^ZJedkm12K;69He$$x$X{sReU65nF=NF!;`q!}TJUPjgk%Gu1O_Mqy#G$FVK{yqzeEUd{*wqDp4V^uha4JS3+_{uB>#C4{09=yB)-M!kw(&> zNi#wcy^O3!c*qeLpa}5(JGqA8_;LIaA;9@hB6N6OzwsY98uJtXS(7CHc^Ui%63`^R z#p;np(x6E*LK3}=TG-*aiqL-2N2oE^|0~7(?e<#;496ydo@*`50jtx z&p#!}e|`)80|{sn-(vMhBWcj286k;YM%E)dEJ$VljJ||g8x7Qn#8wQJ<>=TG-*aiqL-2N2oE^|0~7(?e<#;496ydo@*`2hUIZ=bw}0KYs%Mfdn*(Z?Sr$ku+%1jF3bxBkK_!as&n_0=)lDu3eaQ>4B9iG>3{D=JiTnFy+FG=#BkHLQ+0ZrmttR8734Vp9~B+<*rdW44@fdPsD z@4u647>*yuFA)Nq|0F_(=k*)^A^$(bf7T_*f5PBDkboxfEmn^-k_Jth5t8U-WIe(| zj=%s#fcM|YH4Mj(%o2eN%Ego@E=G(llT^^M;b|kCd~** z^fIy@;UPz0fFi*A@8lYW>LmG3C-@H}pheaQ>4B9iG>3 z{09%7pZL!YljJ{Fg8x7Qn#8wQJ<>=TG-*aiqL-2N2oE^|0~7(?e<#;496ydo@*G{(o)&_jxQy{xcH%2NKXEzQyX1M$({3GeQ!*jI2j^$PpNz2=M+pxrX8R zar_b?!1+%iba-CB@gMU4L;UCQB>9g6{09=yB)-M!kw(&>Ni#wcy^O3!c*qeLpa}5( zJGqA8_;LIaA;9@hB6N6OzwsaP|8pa_&!!~#PYL)BB%n!ri`65Iq(PHrgd}|CtB=0|{sn z-(vMhBWcj286k;YM%E)dB%R z;6IRnCh;v+k2I17O_~vs=w)O*!b6V007ZcJ-^n!$$B*Ne2m#K25~0KM`i=kK!SfUU zc{)k{^F8n%NI;YL7OO`ZNrNWM2ubuZvL4|fM__;=!29py8iwP?@k@jN=Rb+i;d%YW zf5`uj3*6_KB>B(X;6IRnCh;v+k2I17O_~vs=w)O*!b6V007ZcJ-^n!$$B*Ne2m#K2 z5~0KM`i=jP{~zK%KT4ASRD%CN0-D6PSUu868Z>D}NTQdK^#~6+0s|BQ-hU_8FdRRQ zUm^rJ|4D=n&+9k-L;inC!F>Wr@}D)}KahYX@hw)5G?E5Qnh}!dWn?|VLyo`zMS%C; z$u$hekK>mJ0nUFCp~Lg~jsKAUAL2h-lH@<@!G9nDP2yXu9%&>EnlvLM(aXqsgohk~ z0g3?czmsbijvvP_5dxh5BtnPh^%wsk^B-0Q?(=Mt{O2j~A4oux_!g^28cBmD%?L^K zGO`}wAxB_B%T z!G9nDP2yXu9%&>EnlvLM(aXqsgohk~0g3?czmsbijvvP_5dxh5BtnPh^&9^o10XlJ z&+|$0pVz>DAOTI{TdW>wBn_H0BP7ww$a;i_9DxCf0Pnw(YZ#6n$1f2Aoc|<3hv)Sh z{~`ZB#D9LAB>&kB{sReU65nF=NF!;`q!}TJUPjgkST!*KjK zeu)s^{3j7QJg?vQ5BdKg{!^PI|M?L72NKXEzQyX1M$({3GeQ!*jI2j^$PpNz2=M+p zxrX8Rar_b?!1+%iba-BW@gGcoF*Xa_XM2+T=MeZ0B%n!ri`65Iq(PHrgd}(ge(u|Nq zFC*&_9&!W*C<46yPOf1%ejL9<2yp(B2pyi+Z~TW?$ZT+*7n0;Z{|o#F63`^R#p;np z(x6E*LK3}Ni#wcy^O3!c*qeLpa}5(JGqA8_;LIaA;9@hB6N6OzwsaP|1$^N=ch^X zpD6ebB%n!ri`65Iq(PHrgd}KaO7_1UUamgbvT^ zH~vHZf98Vw)FsJ(Qow&80ZrmttR8734Vp9~B+<*rdW44@fdPsD@4u647>*yuFA)Nq z|0F_(=k*)^A^$(be_l+I|73yxKmwY?w^%*WNE$S0Mo6NUk@W}!Fz_EpK$G|ut4A71gC@-gN%S(Z9^oNJV1OdP z`|so$hU3TaON0RDKZ(%cdHu$J@L}>3|M_{6{HGB72NKXEzQyX1M$({3GeQ!*jI2j^ z$PpNz2=M+pxrX8Rar_b?!1+%iba-CB@gHI#H-Y=qC&_;%fd4=On#8wQJ<>=TG-*ai zqL-2N2oE^|0~7(?e<#;496ydo@*`2hUIZ=NC!xpKHK>AOTI{TdW>w zBn_H0BP7ww$a;i_9DxCf0Pnw(YZ#6n$1f2Aoc|<3hv)Sh{~`ZBH-r29GD-e(Blr&_ zphwBn_H0BP7ww$a;i_9DxCf0Pnw(YZ#6n$1f2Aoc|<3hv)Sh|Dgq7 z^TB;~Cdq#mfd4=On#8wQJ<>=TG-*aiqL-2N2oE^|0~7(?e<#;496ydo@*G{(p%7>`IdV+y?#w31|}EV)aNPY0#t@A&Fi_)+0RR2nkST!*KjKeu)s^ z{3j7QJg>j_59kkL9&n$RljJ{lfd4=On#8wQJ<>=TG-*aiqL-2N2oE^|0~7(?e<#;4 z96ydo@*`50jtx&nrptpZmanAOTI{TdW>wBn_H0BP7ww$a;i_9DxCf z0Pnw(YZ#6n$1f2Aoc|<3hv)Sh{~;E#0Nke`N&fRN_zxtYNqmddBaNg%lV*e@dKp=d z@Q@=gKoQ{mcXAEG@#FX=TG-*aiqL-2N2oE^|0~7(?e<#;496ydo@*G{(p%7yp|;Y34s4V z0-D6PSUu868Z>D}NTQdK^#~6+0s|BQ-hU_8FdRRQUm^rJ|4D=n&+9k-L;imjf%`Nj z$$x6We;@%(;#;g9X(SDrG$SO@%gB0!ha7=TG-*aiqL-2N2oE^|0~7(?e<#;496ydo5Kzzaokxvo!G_B+(J~B2KdUIlrEyhX397?2^KX4zv^PDX3=GkE&+xBg{lN8AQRt@<=XTATDzGxhSizf_=U7WR7+D zAEqe3@keUA%l}kgPM0P2_1vUfR)!8!`$$XcBhA`J+Mqtt()&ou=p)V6M_OhdX<2=w z4elc?yN|Rh`bfL7kF+6uq~-LHHnfkl+&N80c{(nj==HnNX2N|Ig^MUuX< zFMk#Fdg7A{_=m)szX~-j?a>d_P4&}YbBhi0ai8#p_{LUAbuIH;E>RW?6 z5-Jz?wsJUY3-U;)-0p8Hhx(pC9to8T_fbx7XIm5QAEMXCjyTDwRzJ&$OV{)D`p!Zg z36+z-tsKr;i98Z2=liyDsBaDONT^((k8*kiTaZWcfhL#KiywSD0(R&tyiB6^txhYo z`rAwM9F71-fFr;W;0SO8I0762jsQo1Bft^h2yg`cZ$qFMWBk<03bc$DN=_O3^CsV%}IL~_RycFM{o%cHk zH)`n=|Dl$C1H#9&bc#Q&rQe8fla@~LC$u=~*OE;AWSr+o?OYUpDi&u>gqyW=ia)KT zyAVF3rBnPzT6!tMfR;}2En50ygsZhU9&tDmS`)us@eu{6^&Q*?Zo0d-T=d^S; z!soSgivL(kpNX(WOQ(3P7DxT$Wa>8y=h?2Ei{d+CaW)&_3tBqGf1;(&LHJWGo#H>! z(&r+q)6yyaqLw}d&tEe2n}_qfq@9=IKiAHC6T*5eo#MaH(rJJDUux;`d)>$F`x?Io znjiIRNv3`tTua+e24%M*si- literal 0 HcmV?d00001 From e67dbfb61f5baef3c178f4e0519fe30ddd52926a Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Wed, 17 Jul 2024 16:22:01 +0800 Subject: [PATCH 6/6] MALI: bifrost: CSF: Add MALI_CSF_INCLUDE_FW to include mali_csffw.bin into driver by default Change-Id: Ib6374444bdda257b245c2f09475cb470f1974dcd Signed-off-by: Zhen Chen --- drivers/gpu/arm/bifrost/Kconfig | 4 +++ .../arm/bifrost/csf/mali_kbase_csf_firmware.c | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/drivers/gpu/arm/bifrost/Kconfig b/drivers/gpu/arm/bifrost/Kconfig index 6880b39323d6..22fdfe80405a 100644 --- a/drivers/gpu/arm/bifrost/Kconfig +++ b/drivers/gpu/arm/bifrost/Kconfig @@ -393,6 +393,10 @@ config MALI_TRACE_POWER_GPU_WORK_PERIOD If unsure, say N. +config MALI_CSF_INCLUDE_FW + depends on MALI_BIFROST && MALI_CSF_SUPPORT + bool "Whether to include CSF firmware into driver" + default y # source "$(MALI_KCONFIG_EXT_PREFIX)drivers/gpu/arm/bifrost/tests/Kconfig" diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c index 8e4608c7e528..952a9b9cdd94 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c @@ -2287,9 +2287,28 @@ int kbase_csf_firmware_late_init(struct kbase_device *kbdev) return 0; } +#ifdef CONFIG_MALI_CSF_INCLUDE_FW +asm ( +" .pushsection .rodata, \"a\" \n" +" .ascii \"CSFFW_ST\" \n" +" .global mali_csffw \n" +"mali_csffw: \n" +" .incbin \"drivers/gpu/arm/bifrost/mali_csffw.bin\" \n" +" .global mali_csffw_end \n" +"mali_csffw_end: \n" +" .ascii \"CSFFW_ED\" \n" +" .popsection \n" +); + +extern char mali_csffw; +extern char mali_csffw_end; +#endif + int kbase_csf_firmware_load_init(struct kbase_device *kbdev) { +#ifndef CONFIG_MALI_CSF_INCLUDE_FW const struct firmware *firmware = NULL; +#endif struct kbase_csf_mcu_fw *const mcu_fw = &kbdev->csf.fw; const u32 magic = FIRMWARE_HEADER_MAGIC; u8 version_major, version_minor; @@ -2297,7 +2316,9 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) u32 entry_end_offset; u32 entry_offset; int ret; +#ifndef CONFIG_MALI_CSF_INCLUDE_FW const char *fw_name = default_fw_name; +#endif lockdep_assert_held(&kbdev->fw_load_lock); @@ -2320,6 +2341,14 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) goto err_out; } +#ifdef CONFIG_MALI_CSF_INCLUDE_FW + mcu_fw->size = &mali_csffw_end - &mali_csffw; + + dev_info(kbdev->dev, "use 'driver built-in firmware' directly\n"); + mcu_fw->data = (u8 *)(&mali_csffw); + dev_dbg(kbdev->dev, "Firmware image (%zu-bytes) retained in csf.fw\n", + mcu_fw->size); +#else #if IS_ENABLED(CONFIG_OF) /* If we can't read CSF firmware name from DTB, * fw_name is not modified and remains the default. @@ -2366,6 +2395,7 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) release_firmware(firmware); } +#endif /* CONFIG_MALI_CSF_INCLUDE_FW */ /* If error in loading or saving the image, branches to error out */ if (ret) @@ -2592,6 +2622,8 @@ void kbase_csf_firmware_unload_term(struct kbase_device *kbdev) kfree(metadata); } + if (IS_ENABLED(CONFIG_MALI_CSF_INCLUDE_FW)) + kbdev->csf.fw.data = NULL; if (kbdev->csf.fw.data) { /* Free the copy of the firmware image */ vfree(kbdev->csf.fw.data);