From 69c7c176552bea612593d3f2b1b032f867fbceb2 Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Tue, 12 Nov 2019 10:48:50 +0800 Subject: [PATCH] MALI: rockchip: upgrade bifrost DDK to r21p0-01rel0, from r20p0-01rel0 Change-Id: Ibdb2508859f24df1014fc485097685bcecb7b31b Signed-off-by: Zhen Chen --- drivers/gpu/arm/bifrost/Kbuild | 2 +- drivers/gpu/arm/bifrost/Kconfig | 14 - drivers/gpu/arm/bifrost/Mconfig | 14 - .../bifrost/backend/gpu/mali_kbase_devfreq.c | 13 +- .../backend/gpu/mali_kbase_device_hw.c | 11 +- .../backend/gpu/mali_kbase_instr_backend.c | 13 +- .../bifrost/backend/gpu/mali_kbase_jm_hw.c | 201 +------------- .../bifrost/backend/gpu/mali_kbase_jm_rb.c | 161 +++-------- .../backend/gpu/mali_kbase_mmu_hw_direct.c | 24 +- .../backend/gpu/mali_kbase_pm_always_on.c | 3 +- .../backend/gpu/mali_kbase_pm_backend.c | 46 ++-- .../backend/gpu/mali_kbase_pm_coarse_demand.c | 3 +- .../bifrost/backend/gpu/mali_kbase_pm_defs.h | 31 ++- .../backend/gpu/mali_kbase_pm_driver.c | 144 +++++----- .../backend/gpu/mali_kbase_pm_internal.h | 38 ++- .../backend/gpu/mali_kbase_pm_policy.c | 60 ++-- .../backend/gpu/mali_kbase_pm_policy.h | 7 +- .../backend/gpu/mali_kbase_pm_shader_states.h | 1 + .../arm/bifrost/backend/gpu/mali_kbase_time.c | 45 --- drivers/gpu/arm/bifrost/build.bp | 3 - drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c | 16 +- .../arm/bifrost/mali_base_hwconfig_features.h | 4 - .../arm/bifrost/mali_base_hwconfig_issues.h | 78 ------ drivers/gpu/arm/bifrost/mali_base_kernel.h | 8 +- drivers/gpu/arm/bifrost/mali_kbase_config.h | 14 +- .../arm/bifrost/mali_kbase_config_defaults.h | 2 - drivers/gpu/arm/bifrost/mali_kbase_context.c | 14 +- .../gpu/arm/bifrost/mali_kbase_core_linux.c | 10 +- .../gpu/arm/bifrost/mali_kbase_ctx_sched.c | 12 +- drivers/gpu/arm/bifrost/mali_kbase_defs.h | 43 +-- drivers/gpu/arm/bifrost/mali_kbase_device.c | 22 -- .../gpu/arm/bifrost/mali_kbase_hwaccess_jm.h | 9 + .../arm/bifrost/mali_kbase_hwaccess_time.h | 22 +- .../gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c | 8 +- drivers/gpu/arm/bifrost/mali_kbase_ioctl.h | 2 +- drivers/gpu/arm/bifrost/mali_kbase_jd.c | 20 -- .../gpu/arm/bifrost/mali_kbase_jd_debugfs.c | 10 +- drivers/gpu/arm/bifrost/mali_kbase_js.c | 29 +- drivers/gpu/arm/bifrost/mali_kbase_mem.c | 84 ++++-- drivers/gpu/arm/bifrost/mali_kbase_mem.h | 20 +- .../gpu/arm/bifrost/mali_kbase_mem_linux.c | 20 +- .../mali_kbase_mem_profile_debugfs_buf_size.h | 2 +- drivers/gpu/arm/bifrost/mali_kbase_mmu.c | 244 +---------------- drivers/gpu/arm/bifrost/mali_kbase_softjobs.c | 4 +- .../gpu/arm/bifrost/mali_kbase_tracepoints.c | 93 ++++++- .../gpu/arm/bifrost/mali_kbase_tracepoints.h | 73 ++++- drivers/gpu/arm/bifrost/mali_kbase_vinstr.c | 2 +- drivers/gpu/arm/bifrost/mali_midg_regmap.h | 5 - drivers/gpu/arm/bifrost/platform/meson/Kbuild | 24 ++ .../platform/meson/mali_kbase_config_meson.c | 41 +++ .../meson/mali_kbase_config_platform.h} | 48 ++-- .../platform/meson/mali_kbase_runtime_pm.c | 257 ++++++++++++++++++ 52 files changed, 886 insertions(+), 1188 deletions(-) create mode 100644 drivers/gpu/arm/bifrost/platform/meson/Kbuild create mode 100644 drivers/gpu/arm/bifrost/platform/meson/mali_kbase_config_meson.c rename drivers/gpu/arm/bifrost/{platform_dummy/mali_ukk_os.h => platform/meson/mali_kbase_config_platform.h} (52%) create mode 100644 drivers/gpu/arm/bifrost/platform/meson/mali_kbase_runtime_pm.c diff --git a/drivers/gpu/arm/bifrost/Kbuild b/drivers/gpu/arm/bifrost/Kbuild index de9456ac58f0..ab12718387a2 100644 --- a/drivers/gpu/arm/bifrost/Kbuild +++ b/drivers/gpu/arm/bifrost/Kbuild @@ -20,7 +20,7 @@ # # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= "r20p0-01rel0" +MALI_RELEASE_NAME ?= "r21p0-01rel0" # Paths required for build KBASE_PATH = $(src) diff --git a/drivers/gpu/arm/bifrost/Kconfig b/drivers/gpu/arm/bifrost/Kconfig index e52d1795906e..a846b0482074 100644 --- a/drivers/gpu/arm/bifrost/Kconfig +++ b/drivers/gpu/arm/bifrost/Kconfig @@ -106,20 +106,6 @@ config MALI_CORESTACK If unsure, say N. -config MALI_PLATFORM_POWER_DOWN_ONLY - bool "Support disabling the power down of individual cores" - depends on MALI_BIFROST && MALI_BIFROST_EXPERT - default n - help - Enabling this feature will let the driver avoid power down of the - shader cores, the tiler, and the L2 cache. - The entire GPU would be powered down at once through the platform - specific code. - This may be required for certain platform configurations only. - This also limits the available power policies. - - If unsure, say N. - config MALI_BIFROST_DEBUG bool "Debug build" depends on MALI_BIFROST && MALI_BIFROST_EXPERT diff --git a/drivers/gpu/arm/bifrost/Mconfig b/drivers/gpu/arm/bifrost/Mconfig index 09bf4c52a531..dc6dfac135ab 100644 --- a/drivers/gpu/arm/bifrost/Mconfig +++ b/drivers/gpu/arm/bifrost/Mconfig @@ -104,20 +104,6 @@ config MALI_CORESTACK If unsure, say N. -config MALI_PLATFORM_POWER_DOWN_ONLY - bool "Support disabling the power down of individual cores" - depends on MALI_BIFROST && MALI_BIFROST_EXPERT - default n - help - Enabling this feature will let the driver avoid power down of the - shader cores, the tiler, and the L2 cache. - The entire GPU would be powered down at once through the platform - specific code. - This may be required for certain platform configurations only. - This also limits the available power policies. - - If unsure, say N. - config MALI_BIFROST_DEBUG bool "Debug build" depends on MALI_BIFROST && MALI_BIFROST_EXPERT diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c index 4d4f4c223e9d..aac9172537cd 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c @@ -472,10 +472,7 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) if (of_property_read_u64(node, "opp-core-mask", &core_mask)) core_mask = shader_present; - if (core_mask != shader_present && - (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11056) || - corestack_driver_control || - platform_power_down_only)) { + if (core_mask != shader_present && corestack_driver_control) { dev_warn(kbdev->dev, "Ignoring OPP %llu - Dynamic Core Scaling not supported on this GPU\n", opp_freq); @@ -696,7 +693,9 @@ int kbase_devfreq_init(struct kbase_device *kbdev) "simple_ondemand", NULL); if (IS_ERR(kbdev->devfreq)) { err = PTR_ERR(kbdev->devfreq); - goto add_device_failed; + kbase_devfreq_work_term(kbdev); + kbase_devfreq_term_freq_table(kbdev); + return err; } /* devfreq_add_device only copies a few of kbdev->dev's fields, so @@ -754,10 +753,8 @@ opp_notifier_failed: dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); else kbdev->devfreq = NULL; -add_device_failed: - kbase_devfreq_work_term(kbdev); - kbase_devfreq_term_freq_table(kbdev); + kbase_devfreq_work_term(kbdev); return err; } diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_device_hw.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_device_hw.c index ccc1ee2de9d0..958d8d5c58be 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_device_hw.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_device_hw.c @@ -346,20 +346,13 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) if (val & POWER_CHANGED_ALL) { kbase_pm_power_changed(kbdev); } else if (val & CLEAN_CACHES_COMPLETED) { - /* When 'platform_power_down_only' is enabled, the L2 cache is - * not powered down, but flushed before the GPU power down - * (which is done by the platform code). So the L2 state machine - * requests a cache flush. And when that flush completes, the L2 - * state machine needs to be re-invoked to proceed with the GPU - * power down. - * If cache line evict messages can be lost when shader cores + /* If cache line evict messages can be lost when shader cores * power down then we need to flush the L2 cache before powering * down cores. When the flush completes, the shaders' state * machine needs to be re-invoked to proceed with powering down * cores. */ - if (platform_power_down_only || - kbdev->pm.backend.l2_always_on || + if (kbdev->pm.backend.l2_always_on || kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) kbase_pm_power_changed(kbdev); } diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c index 1d18326ea9eb..4e216d32257e 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c @@ -88,21 +88,12 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, enable->mmu_l2_bm); /* Due to PRLAM-8186 we need to disable the Tiler before we enable the * HW counter dump. */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) - kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0); - else - kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), - enable->tiler_bm); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), + enable->tiler_bm); kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL); - /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump - */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186)) - kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), - enable->tiler_bm); - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c index 323f0f229c91..b3093a6014ec 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c @@ -140,9 +140,6 @@ void kbase_job_hw_submit(struct kbase_device *kbdev, else cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10649)) - cfg |= JS_CONFIG_START_MMU; - cfg |= JS_CONFIG_THREAD_PRI(8); if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE) && @@ -444,19 +441,6 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) done = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) { - /* Workaround for missing interrupt caused by - * PRLAM-10883 */ - if (((active >> i) & 1) && (0 == - kbase_reg_read(kbdev, - JOB_SLOT_REG(i, - JS_STATUS)))) { - /* Force job slot to be processed again - */ - done |= (1u << i); - } - } - failed = done >> 16; finished = (done & 0xFFFF) | failed; if (done) @@ -478,32 +462,6 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count); } -static bool kbasep_soft_stop_allowed(struct kbase_device *kbdev, - struct kbase_jd_atom *katom) -{ - bool soft_stops_allowed = true; - - if (kbase_jd_katom_is_protected(katom)) { - soft_stops_allowed = false; - } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) { - if ((katom->core_req & BASE_JD_REQ_T) != 0) - soft_stops_allowed = false; - } - return soft_stops_allowed; -} - -static bool kbasep_hard_stop_allowed(struct kbase_device *kbdev, - base_jd_core_req core_reqs) -{ - bool hard_stops_allowed = true; - - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) { - if ((core_reqs & BASE_JD_REQ_T) != 0) - hard_stops_allowed = false; - } - return hard_stops_allowed; -} - void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, int js, u32 action, @@ -527,13 +485,10 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, #endif if (action == JS_COMMAND_SOFT_STOP) { - bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev, - target_katom); - - if (!soft_stop_allowed) { + if (kbase_jd_katom_is_protected(target_katom)) { #ifdef CONFIG_MALI_BIFROST_DEBUG dev_dbg(kbdev->dev, - "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X", + "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%x", (unsigned int)core_reqs); #endif /* CONFIG_MALI_BIFROST_DEBUG */ return; @@ -546,38 +501,6 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, /* Mark the point where we issue the soft-stop command */ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(kbdev, target_katom); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { - int i; - - for (i = 0; - i < kbase_backend_nr_atoms_submitted(kbdev, js); - i++) { - struct kbase_jd_atom *katom; - - katom = kbase_gpu_inspect(kbdev, js, i); - - KBASE_DEBUG_ASSERT(katom); - - /* For HW_ISSUE_8316, only 'bad' jobs attacking - * the system can cause this issue: normally, - * all memory should be allocated in multiples - * of 4 pages, and growable memory should be - * changed size in multiples of 4 pages. - * - * Whilst such 'bad' jobs can be cleared by a - * GPU reset, the locking up of a uTLB entry - * caused by the bad job could also stall other - * ASs, meaning that other ASs' jobs don't - * complete in the 'grace' period before the - * reset. We don't want to lose other ASs' jobs - * when they would normally complete fine, so we - * must 'poke' the MMU regularly to help other - * ASs complete */ - kbase_as_poking_timer_retain_atom( - kbdev, katom->kctx, katom); - } - } - if (kbase_hw_has_feature( kbdev, BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { @@ -587,34 +510,6 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, JS_COMMAND_SOFT_STOP_0; } } else if (action == JS_COMMAND_HARD_STOP) { - bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev, - core_reqs); - - if (!hard_stop_allowed) { - /* Jobs can be hard-stopped for the following reasons: - * * CFS decides the job has been running too long (and - * soft-stop has not occurred). In this case the GPU - * will be reset by CFS if the job remains on the - * GPU. - * - * * The context is destroyed, kbase_jd_zap_context - * will attempt to hard-stop the job. However it also - * has a watchdog which will cause the GPU to be - * reset if the job remains on the GPU. - * - * * An (unhandled) MMU fault occurred. As long as - * BASE_HW_ISSUE_8245 is defined then the GPU will be - * reset. - * - * All three cases result in the GPU being reset if the - * hard-stop fails, so it is safe to just return and - * ignore the hard-stop request. - */ - dev_warn(kbdev->dev, - "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X", - (unsigned int)core_reqs); - return; - } target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED; if (kbase_hw_has_feature( @@ -835,63 +730,6 @@ void kbase_job_slot_term(struct kbase_device *kbdev) } KBASE_EXPORT_TEST_API(kbase_job_slot_term); -/** - * kbasep_check_for_afbc_on_slot() - Check whether AFBC is in use on this slot - * @kbdev: kbase device pointer - * @kctx: context to check against - * @js: slot to check - * @target_katom: An atom to check, or NULL if all atoms from @kctx on - * slot @js should be checked - * - * This checks are based upon parameters that would normally be passed to - * kbase_job_slot_hardstop(). - * - * In the event of @target_katom being NULL, this will check the last jobs that - * are likely to be running on the slot to see if a) they belong to kctx, and - * so would be stopped, and b) whether they have AFBC - * - * In that case, It's guaranteed that a job currently executing on the HW with - * AFBC will be detected. However, this is a conservative check because it also - * detects jobs that have just completed too. - * - * Return: true when hard-stop _might_ stop an afbc atom, else false. - */ -static bool kbasep_check_for_afbc_on_slot(struct kbase_device *kbdev, - struct kbase_context *kctx, int js, - struct kbase_jd_atom *target_katom) -{ - bool ret = false; - int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - /* When we have an atom the decision can be made straight away. */ - if (target_katom) - return !!(target_katom->core_req & BASE_JD_REQ_FS_AFBC); - - /* Otherwise, we must chweck the hardware to see if it has atoms from - * this context with AFBC. */ - for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) { - struct kbase_jd_atom *katom; - - katom = kbase_gpu_inspect(kbdev, js, i); - if (!katom) - continue; - - /* Ignore atoms from other contexts, they won't be stopped when - * we use this for checking if we should hard-stop them */ - if (katom->kctx != kctx) - continue; - - /* An atom on this slot and this context: check for AFBC */ - if (katom->core_req & BASE_JD_REQ_FS_AFBC) { - ret = true; - break; - } - } - - return ret; -} /** * kbase_job_slot_softstop_swflags - Soft-stop a job with flags @@ -948,31 +786,10 @@ void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, { struct kbase_device *kbdev = kctx->kbdev; bool stopped; - /* We make the check for AFBC before evicting/stopping atoms. Note - * that no other thread can modify the slots whilst we have the - * hwaccess_lock. */ - int needs_workaround_for_afbc = - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3542) - && kbasep_check_for_afbc_on_slot(kbdev, kctx, js, - target_katom); stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js, target_katom, JS_COMMAND_HARD_STOP); - if (stopped && (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) || - kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) || - needs_workaround_for_afbc)) { - /* MIDBASE-2916 if a fragment job with AFBC encoding is - * hardstopped, ensure to do a soft reset also in order to - * clear the GPU status. - * Workaround for HW issue 8401 has an issue,so after - * hard-stopping just reset the GPU. This will ensure that the - * jobs leave the GPU.*/ - if (kbase_prepare_to_reset_gpu_locked(kbdev)) { - dev_err(kbdev->dev, "Issueing GPU soft-reset after hard stopping due to hardware issue"); - kbase_reset_gpu_locked(kbdev); - } - } } /** @@ -982,7 +799,7 @@ void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, * @core_reqs: core requirements of the atom * @target_katom: the atom which is being affected * - * For a certain soft/hard-stop action, work out whether to enter disjoint + * For a certain soft-stop action, work out whether to enter disjoint * state. * * This does not register multiple disjoint events if the atom has already @@ -1000,16 +817,12 @@ void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, { u32 hw_action = action & JS_COMMAND_MASK; - /* For hard-stop, don't enter if hard-stop not allowed */ - if (hw_action == JS_COMMAND_HARD_STOP && - !kbasep_hard_stop_allowed(kbdev, core_reqs)) - return; - /* For soft-stop, don't enter if soft-stop not allowed, or isn't - * causing disjoint */ + * causing disjoint. + */ if (hw_action == JS_COMMAND_SOFT_STOP && - !(kbasep_soft_stop_allowed(kbdev, target_katom) && - (action & JS_COMMAND_SW_CAUSES_DISJOINT))) + (kbase_jd_katom_is_protected(target_katom) || + (0 == (action & JS_COMMAND_SW_CAUSES_DISJOINT)))) return; /* Nothing to do if already logged disjoint state on this atom */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c index 55440b84596e..a1f90f5b8c6a 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c @@ -132,42 +132,7 @@ struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; } -/** - * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently - * on the GPU - * @kbdev: Device pointer - * @js: Job slot to inspect - * - * Return: true if there are atoms on the GPU for slot js, - * false otherwise - */ -static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js) -{ - int i; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - for (i = 0; i < SLOT_RB_SIZE; i++) { - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); - - if (!katom) - return false; - if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED || - katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY) - return true; - } - - return false; -} - -/** - * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms - * currently on the GPU - * @kbdev: Device pointer - * - * Return: true if there are any atoms on the GPU, false otherwise - */ -static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) +bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) { int js; int i; @@ -420,40 +385,6 @@ static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; } -static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js) -{ - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - bool slot_busy[3]; - - if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) - return true; - slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0, - KBASE_ATOM_GPU_RB_WAITING_AFFINITY); - slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1, - KBASE_ATOM_GPU_RB_WAITING_AFFINITY); - slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2, - KBASE_ATOM_GPU_RB_WAITING_AFFINITY); - - if ((js == 2 && !(slot_busy[0] || slot_busy[1])) || - (js != 2 && !slot_busy[2])) - return true; - - /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */ - if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) || - kbase_gpu_atoms_submitted(kbdev, 1) || - backend->rmu_workaround_flag)) - return false; - - /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */ - if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) || - !backend->rmu_workaround_flag)) - return false; - - backend->rmu_workaround_flag = !backend->rmu_workaround_flag; - - return true; -} - /** * other_slots_busy - Determine if any job slots other than @js are currently * running atoms @@ -625,8 +556,8 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, kbdev->protected_mode_transition = true; - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_ENTER_PROTECTED_HWCNT: /* See if we can get away with disabling hwcnt atomically */ kbdev->protected_mode_hwcnt_desired = false; @@ -672,8 +603,8 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, if (!kbdev->pm.backend.protected_entry_transition_override) kbase_pm_update_cores_state_nolock(kbdev); - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: /* Avoid unnecessary waiting on non-ACE platforms. */ if (kbdev->system_coherency == COHERENCY_ACE) { @@ -699,8 +630,8 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY; - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: /* * When entering into protected mode, we must ensure that the @@ -732,8 +663,8 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) return -EAGAIN; - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_ENTER_PROTECTED_FINISHED: if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { /* @@ -802,7 +733,8 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, kbase_pm_protected_override_enable(kbdev); kbase_pm_update_cores_state_nolock(kbdev); - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: if (kbdev->pm.backend.l2_state != KBASE_L2_OFF) { /* @@ -814,8 +746,8 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_RESET; - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_EXIT_PROTECTED_RESET: /* Issue the reset to the GPU */ err = kbase_gpu_protected_mode_reset(kbdev); @@ -856,8 +788,8 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: /* A GPU reset is issued when exiting protected mode. Once the * reset is done all atoms' state will also be reset. For this @@ -909,8 +841,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: if (kbase_gpu_check_secure_atoms(kbdev, !kbase_jd_katom_is_protected( @@ -929,8 +861,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: /* @@ -964,8 +896,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: if (katom[idx]->will_fail_event_code) { kbase_gpu_mark_atom_for_return(kbdev, @@ -1001,17 +933,14 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_AFFINITY; - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: - if (!kbase_gpu_rmu_workaround(kbdev, js)) - break; - katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_GPU_RB_READY: if (idx == 1) { @@ -1059,8 +988,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) kbase_pm_metrics_update(kbdev, &katom[idx]->start_timestamp); - /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ - + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ case KBASE_ATOM_GPU_RB_SUBMITTED: /* Atom submitted to HW, nothing else to do */ break; @@ -1079,12 +1008,6 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) } } } - - /* Warn if PRLAM-8987 affinity restrictions are violated */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) - WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) || - kbase_gpu_atoms_submitted(kbdev, 1)) && - kbase_gpu_atoms_submitted(kbdev, 2)); } @@ -1164,8 +1087,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, } } - if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) || (katom->core_req & - BASE_JD_REQ_SKIP_CACHE_END)) && + if ((katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && completion_code != BASE_JD_EVENT_DONE && !(completion_code & BASE_JD_SW_EVENT)) { /* When a job chain fails, on a T60x or when @@ -1174,12 +1096,6 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, * corruption we need to flush the cache manually before any * affected memory gets reused. */ katom->need_cache_flush_cores_retained = true; - } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) { - if (kbdev->gpu_props.num_core_groups > 1 && - katom->device_nr >= 1) { - dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n"); - katom->need_cache_flush_cores_retained = true; - } } katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); @@ -1343,6 +1259,9 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); } + /* For partial shader core off L2 cache flush */ + kbase_pm_update_state(kbdev); + /* Job completion may have unblocked other atoms. Try to update all job * slots */ kbase_backend_slot_update(kbdev); @@ -1687,24 +1606,6 @@ void kbase_backend_complete_wq(struct kbase_device *kbdev, * now */ kbase_backend_cache_clean(kbdev, katom); - - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) && - (katom->core_req & BASE_JD_REQ_FS) && - katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT && - (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) && - !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) { - dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n"); - if (kbasep_10969_workaround_clamp_coordinates(katom)) { - /* The job had a TILE_RANGE_FAULT after was soft-stopped - * Due to an HW issue we try to execute the job again. - */ - dev_dbg(kbdev->dev, - "Clamping has been executed, try to rerun the job\n" - ); - katom->event_code = BASE_JD_EVENT_STOPPED; - katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN; - } - } } void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_mmu_hw_direct.c index 77e0b78f0993..670e1cbf1c9c 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_mmu_hw_direct.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_mmu_hw_direct.c @@ -48,7 +48,7 @@ static inline u64 lock_region(struct kbase_device *kbdev, u64 pfn, /* gracefully handle num_pages being zero */ if (0 == num_pages) { - region |= 11; + region |= KBASE_LOCK_REGION_MIN_SIZE; } else { u8 region_width; @@ -57,8 +57,9 @@ static inline u64 lock_region(struct kbase_device *kbdev, u64 pfn, /* not pow2, so must go up to the next pow2 */ region_width += 1; } + region_width = MAX(region_width, KBASE_LOCK_REGION_MIN_SIZE); + KBASE_DEBUG_ASSERT(region_width <= KBASE_LOCK_REGION_MAX_SIZE); - KBASE_DEBUG_ASSERT(region_width >= KBASE_LOCK_REGION_MIN_SIZE); region |= region_width; } @@ -313,25 +314,6 @@ int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, /* Wait for the flush to complete */ ret = wait_ready(kbdev, as->number); - - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9630)) { - /* Issue an UNLOCK command to ensure that valid page - tables are re-read by the GPU after an update. - Note that, the FLUSH command should perform all the - actions necessary, however the bus logs show that if - multiple page faults occur within an 8 page region - the MMU does not always re-read the updated page - table entries for later faults or is only partially - read, it subsequently raises the page fault IRQ for - the same addresses, the unlock ensures that the MMU - cache is flushed, so updates can be re-read. As the - region is now unlocked we need to issue 2 UNLOCK - commands in order to flush the MMU/uTLB, - see PRLAM-8812. - */ - write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK); - write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK); - } } return ret; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c index 51a10a231df0..e33fe0b8e415 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_always_on.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2015,2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -61,7 +61,6 @@ const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { always_on_term, /* term */ always_on_shaders_needed, /* shaders_needed */ always_on_get_core_active, /* get_core_active */ - 0u, /* flags */ KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ }; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c index cf28d115d0f7..be3348870151 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c @@ -65,6 +65,8 @@ int kbase_pm_runtime_init(struct kbase_device *kbdev) callbacks->power_runtime_off_callback; kbdev->pm.backend.callback_power_runtime_idle = callbacks->power_runtime_idle_callback; + kbdev->pm.backend.callback_soft_reset = + callbacks->soft_reset_callback; if (callbacks->power_runtime_init_callback) return callbacks->power_runtime_init_callback(kbdev); @@ -81,6 +83,7 @@ int kbase_pm_runtime_init(struct kbase_device *kbdev) kbdev->pm.backend.callback_power_runtime_on = NULL; kbdev->pm.backend.callback_power_runtime_off = NULL; kbdev->pm.backend.callback_power_runtime_idle = NULL; + kbdev->pm.backend.callback_soft_reset = NULL; return 0; } @@ -158,8 +161,7 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) if (kbase_pm_ca_init(kbdev) != 0) goto workq_fail; - if (kbase_pm_policy_init(kbdev) != 0) - goto pm_policy_fail; + kbase_pm_policy_init(kbdev); if (kbase_pm_state_machine_init(kbdev) != 0) goto pm_state_machine_fail; @@ -170,16 +172,6 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) kbase_pm_hwcnt_disable_worker); kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); - /* At runtime, this feature can be enabled via module parameter - * when insmod is executed. Then this will override all workarounds. - */ - if (platform_power_down_only) { - kbdev->pm.backend.gpu_clock_slow_down_wa = false; - kbdev->pm.backend.l2_always_on = false; - - return 0; - } - if (IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED)) { kbdev->pm.backend.l2_always_on = false; kbdev->pm.backend.gpu_clock_slow_down_wa = false; @@ -214,7 +206,6 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) pm_state_machine_fail: kbase_pm_policy_term(kbdev); -pm_policy_fail: kbase_pm_ca_term(kbdev); workq_fail: kbasep_pm_metrics_term(kbdev); @@ -256,11 +247,10 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) struct kbasep_js_device_data *js_devdata = &kbdev->js_data; unsigned long flags; - if (!platform_power_down_only) - /* Wait for power transitions to complete. We do this with no locks held - * so that we don't deadlock with any pending workqueues. - */ - kbase_pm_wait_for_desired_state(kbdev); + /* Wait for power transitions to complete. We do this with no locks held + * so that we don't deadlock with any pending workqueues. + */ + kbase_pm_wait_for_desired_state(kbdev); mutex_lock(&js_devdata->runpool_mutex); mutex_lock(&kbdev->pm.lock); @@ -275,7 +265,7 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* Disable interrupts and turn the clock off */ - if (!kbase_pm_clock_off(kbdev, backend->poweroff_is_suspend)) { + if (!kbase_pm_clock_off(kbdev)) { /* * Page/bus faults are pending, must drop locks to * process. Interrupts are disabled so no more faults @@ -296,8 +286,7 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) if (backend->poweron_required) kbase_pm_clock_on(kbdev, false); else - WARN_ON(!kbase_pm_clock_off(kbdev, - backend->poweroff_is_suspend)); + WARN_ON(!kbase_pm_clock_off(kbdev)); } } @@ -473,7 +462,7 @@ static void kbase_pm_hwcnt_disable_worker(struct work_struct *data) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } -void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend) +void kbase_pm_do_poweroff(struct kbase_device *kbdev) { unsigned long flags; @@ -492,7 +481,6 @@ void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend) kbdev->pm.backend.l2_desired = false; kbdev->pm.backend.poweroff_wait_in_progress = true; - kbdev->pm.backend.poweroff_is_suspend = is_suspend; kbdev->pm.backend.invoke_poweroff_wait_wq_when_l2_off = true; /* l2_desired being false should cause the state machine to @@ -587,7 +575,7 @@ void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) KBASE_DEBUG_ASSERT(kbdev != NULL); mutex_lock(&kbdev->pm.lock); - kbase_pm_do_poweroff(kbdev, false); + kbase_pm_do_poweroff(kbdev); mutex_unlock(&kbdev->pm.lock); } @@ -636,13 +624,16 @@ void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask_js0, u64 new_core_mask_js1, u64 new_core_mask_js2) { + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&kbdev->pm.lock); + kbdev->pm.debug_core_mask[0] = new_core_mask_js0; kbdev->pm.debug_core_mask[1] = new_core_mask_js1; kbdev->pm.debug_core_mask[2] = new_core_mask_js2; kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 | new_core_mask_js2; - kbase_pm_update_cores_state_nolock(kbdev); + kbase_pm_update_dynamic_cores_onoff(kbdev); } void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) @@ -665,7 +656,7 @@ void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) mutex_lock(&js_devdata->runpool_mutex); mutex_lock(&kbdev->pm.lock); - kbase_pm_do_poweroff(kbdev, true); + kbase_pm_do_poweroff(kbdev); kbase_backend_timer_suspend(kbdev); @@ -673,6 +664,9 @@ void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) mutex_unlock(&js_devdata->runpool_mutex); kbase_pm_wait_for_poweroff_complete(kbdev); + + if (kbdev->pm.backend.callback_power_suspend) + kbdev->pm.backend.callback_power_suspend(kbdev); } void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c index e90c44def25e..9eef44ad877f 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_coarse_demand.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2012-2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -60,7 +60,6 @@ const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { coarse_demand_term, /* term */ coarse_demand_shaders_needed, /* shaders_needed */ coarse_demand_get_core_active, /* get_core_active */ - 0u, /* flags */ KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ }; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h index 42835ddedcf7..45ea1de9fbbc 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h @@ -37,11 +37,6 @@ struct kbase_device; struct kbase_jd_atom; -/** - * Maximum number of PM policies that may be active on a device. - */ -#define KBASE_PM_MAX_NUM_POLICIES (10) - /** * enum kbase_pm_core_type - The types of core in a GPU. * @@ -100,7 +95,9 @@ enum kbase_l2_core_state { * been requested to power on and hwcnt * is being disabled * @KBASE_SHADERS_PEND_ON_CORESTACK_ON: Core stacks are on, shaders have been - * requested to power on. + * requested to power on. Or after doing + * partial shader on/off, checking whether + * it's the desired state. * @KBASE_SHADERS_ON_CORESTACK_ON: The shaders and core stacks are on, and hwcnt * already enabled. * @KBASE_SHADERS_ON_CORESTACK_ON_RECHECK: The shaders and core stacks @@ -110,6 +107,10 @@ enum kbase_l2_core_state { * @KBASE_SHADERS_WAIT_OFF_CORESTACK_ON: The shaders have been requested to * power off, but they remain on for the * duration of the hysteresis timer + * @KBASE_SHADERS_WAIT_GPU_IDLE: The shaders partial poweroff needs to reach + * a state where jobs on the GPU are finished + * including jobs currently running and in the + * GPU queue because of GPU2017-861 * @KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON: The hysteresis timer has expired * @KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON: The core stacks are on and the * level 2 cache is being flushed. @@ -273,8 +274,6 @@ union kbase_pm_policy_data { * when poweroff_wait_in_progress is true, and therefore the * GPU can not immediately be powered on. pm.lock must be * held when accessing - * @poweroff_is_suspend: true if the GPU is being powered off due to a suspend - * request. pm.lock must be held when accessing * @gpu_poweroff_wait_wq: workqueue for waiting for GPU to power off * @gpu_poweroff_wait_work: work item for use with @gpu_poweroff_wait_wq * @poweroff_wait: waitqueue for waiting for @gpu_poweroff_wait_work to complete @@ -292,6 +291,8 @@ union kbase_pm_policy_data { * &struct kbase_pm_callback_conf * @callback_power_runtime_idle: Optional callback when the GPU may be idle. See * &struct kbase_pm_callback_conf + * @callback_soft_reset: Optional callback to software reset the GPU. See + * &struct kbase_pm_callback_conf * @ca_cores_enabled: Cores that are currently available * @l2_state: The current state of the L2 cache state machine. See * &enum kbase_l2_core_state @@ -311,6 +312,10 @@ union kbase_pm_policy_data { * that the policy doesn't change its mind in the mean time). * @in_reset: True if a GPU is resetting and normal power manager operation is * suspended + * @partial_shaderoff: True if we want to partial power off shader cores, + * it indicates a partial shader core off case, + * do some special operation for such case like flush + * L2 cache because of GPU2017-861 * @protected_entry_transition_override : True if GPU reset is being used * before entering the protected mode and so * the reset handling behaviour is being @@ -376,7 +381,6 @@ struct kbase_pm_backend_data { bool poweroff_wait_in_progress; bool invoke_poweroff_wait_wq_when_l2_off; bool poweron_required; - bool poweroff_is_suspend; struct workqueue_struct *gpu_poweroff_wait_wq; struct work_struct gpu_poweroff_wait_work; @@ -390,6 +394,7 @@ struct kbase_pm_backend_data { int (*callback_power_runtime_on)(struct kbase_device *kbdev); void (*callback_power_runtime_off)(struct kbase_device *kbdev); int (*callback_power_runtime_idle)(struct kbase_device *kbdev); + int (*callback_soft_reset)(struct kbase_device *kbdev); u64 ca_cores_enabled; @@ -402,6 +407,8 @@ struct kbase_pm_backend_data { bool in_reset; + bool partial_shaderoff; + bool protected_entry_transition_override; bool protected_transition_override; int protected_l2_override; @@ -427,10 +434,6 @@ enum kbase_pm_policy_id { KBASE_PM_POLICY_ID_ALWAYS_ON }; -typedef u32 kbase_pm_policy_flags; - -#define KBASE_PM_POLICY_FLAG_DISABLED_WITH_POWER_DOWN_ONLY (1u) - /** * struct kbase_pm_policy - Power policy structure. * @@ -443,7 +446,6 @@ typedef u32 kbase_pm_policy_flags; * @shaders_needed: Function called to find out if shader cores are needed * @get_core_active: Function called to get the current overall GPU power * state - * @flags: Field indicating flags for this policy * @id: Field indicating an ID for this policy. This is not * necessarily the same as its index in the list returned * by kbase_pm_list_policies(). @@ -500,7 +502,6 @@ struct kbase_pm_policy { */ bool (*get_core_active)(struct kbase_device *kbdev); - kbase_pm_policy_flags flags; enum kbase_pm_policy_id id; }; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c index 92d3818305cc..ff4f574d06db 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c @@ -58,15 +58,6 @@ MODULE_PARM_DESC(corestack_driver_control, "to the Mali GPU is known to be problematic."); KBASE_EXPORT_TEST_API(corestack_driver_control); -#ifdef CONFIG_MALI_PLATFORM_POWER_DOWN_ONLY -bool platform_power_down_only = true; -#else -bool platform_power_down_only; /* Default value of 0/false */ -#endif -module_param(platform_power_down_only, bool, 0444); -MODULE_PARM_DESC(platform_power_down_only, - "Disable power down of individual cores."); - /** * enum kbasep_pm_action - Actions that can be performed on a core. * @@ -95,7 +86,7 @@ static u64 kbase_pm_get_state( enum kbase_pm_core_type core_type, enum kbasep_pm_action action); -static bool kbase_pm_is_l2_desired(struct kbase_device *kbdev) +bool kbase_pm_is_l2_desired(struct kbase_device *kbdev) { if (kbdev->pm.backend.protected_entry_transition_override) return false; @@ -268,12 +259,6 @@ static void kbase_pm_invoke(struct kbase_device *kbdev, u32 lo = cores & 0xFFFFFFFF; u32 hi = (cores >> 32) & 0xFFFFFFFF; - /* When 'platform_power_down_only' is enabled, no core type should be - * turned off individually. - */ - KBASE_DEBUG_ASSERT(!(action == ACTION_PWROFF && - platform_power_down_only)); - lockdep_assert_held(&kbdev->hwaccess_lock); reg = core_type_to_reg(core_type, action); @@ -764,7 +749,7 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev) break; case KBASE_L2_POWER_DOWN: - if (!platform_power_down_only && !backend->l2_always_on) + if (!backend->l2_always_on) /* Powering off the L2 will also power off the * tiler. */ @@ -787,7 +772,7 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev) break; case KBASE_L2_PEND_OFF: - if (!platform_power_down_only && !backend->l2_always_on) { + if (!backend->l2_always_on) { /* We only need to check the L2 here - if the L2 * is off then the tiler is definitely also off. */ @@ -992,6 +977,7 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) kbdev->hwcnt_gpu_ctx); backend->hwcnt_disabled = false; } + backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON; } break; @@ -1001,7 +987,7 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) /* If shaders to change state, trigger a counter dump */ if (!backend->shaders_desired || - (backend->shaders_avail & ~shaders_ready)) { + (backend->shaders_avail != shaders_ready)) { backend->hwcnt_desired = false; if (!backend->hwcnt_disabled) kbase_pm_trigger_hwcnt_disable(kbdev); @@ -1045,15 +1031,24 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) backend->shaders_state = KBASE_SHADERS_WAIT_OFF_CORESTACK_ON; } - } else { - if (backend->shaders_avail & ~shaders_ready) { - backend->shaders_avail |= shaders_ready; + } else if (backend->shaders_avail & ~shaders_ready) { + /* set cores ready but not available to + * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON + * check pass + */ + backend->shaders_avail |= shaders_ready; - kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, - backend->shaders_avail & ~shaders_ready, - ACTION_PWRON); - } - backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; + kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, + backend->shaders_avail & ~shaders_ready, + ACTION_PWRON); + backend->shaders_state = + KBASE_SHADERS_PEND_ON_CORESTACK_ON; + } else if (shaders_ready & ~backend->shaders_avail) { + backend->shaders_state = + KBASE_SHADERS_WAIT_GPU_IDLE; + } else { + backend->shaders_state = + KBASE_SHADERS_PEND_ON_CORESTACK_ON; } break; @@ -1071,8 +1066,20 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) } break; + case KBASE_SHADERS_WAIT_GPU_IDLE: + /* If partial shader core off need to wait the job in + * running and next register finished then flush L2 + * or it might hit GPU2017-861 + */ + if (!kbase_gpu_atoms_submitted_any(kbdev)) { + backend->partial_shaderoff = true; + backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; + } + break; + case KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON: shader_poweroff_timer_queue_cancel(kbdev); + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) { kbase_gpu_start_cache_clean_nolock(kbdev); backend->shaders_state = @@ -1091,20 +1098,35 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) break; case KBASE_SHADERS_READY_OFF_CORESTACK_ON: - if (!platform_power_down_only) + if (backend->partial_shaderoff) { + backend->partial_shaderoff = false; + /* remove cores available but not ready to + * meet KBASE_SHADERS_PEND_ON_CORESTACK_ON + * check pass + */ + backend->shaders_avail &= shaders_ready; + kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, + shaders_ready & ~backend->shaders_avail, ACTION_PWROFF); + backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; + KBASE_TRACE_ADD(kbdev, + PM_CORES_CHANGE_AVAILABLE, + NULL, NULL, 0u, + (u32)(shaders_ready & ~backend->shaders_avail)); + } else { kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, shaders_ready, ACTION_PWROFF); - KBASE_TRACE_ADD(kbdev, - PM_CORES_CHANGE_AVAILABLE, - NULL, NULL, 0u, 0u); + KBASE_TRACE_ADD(kbdev, + PM_CORES_CHANGE_AVAILABLE, + NULL, NULL, 0u, 0u); - backend->shaders_state = KBASE_SHADERS_PEND_OFF_CORESTACK_ON; + backend->shaders_state = KBASE_SHADERS_PEND_OFF_CORESTACK_ON; + } break; case KBASE_SHADERS_PEND_OFF_CORESTACK_ON: - if ((!shaders_trans && !shaders_ready) || platform_power_down_only) { - if (corestack_driver_control && !platform_power_down_only) + if (!shaders_trans && !shaders_ready) { + if (corestack_driver_control) kbase_pm_invoke(kbdev, KBASE_PM_CORE_STACK, stacks_avail, ACTION_PWROFF); @@ -1113,8 +1135,7 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev) break; case KBASE_SHADERS_OFF_CORESTACK_PEND_OFF: - if ((!stacks_trans && !stacks_ready) || - platform_power_down_only) { + if (!stacks_trans && !stacks_ready) { /* On powered off, re-enable the hwcnt */ backend->pm_shaders_core_mask = 0; backend->hwcnt_desired = true; @@ -1571,7 +1592,7 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) KBASE_EXPORT_TEST_API(kbase_pm_clock_on); -bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend) +bool kbase_pm_clock_off(struct kbase_device *kbdev) { unsigned long flags; @@ -1585,8 +1606,6 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend) if (!kbdev->pm.backend.gpu_powered) { /* Already turned off */ - if (is_suspend && kbdev->pm.backend.callback_power_suspend) - kbdev->pm.backend.callback_power_suspend(kbdev); return true; } @@ -1612,9 +1631,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend) kbdev->pm.backend.gpu_powered = false; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (is_suspend && kbdev->pm.backend.callback_power_suspend) - kbdev->pm.backend.callback_power_suspend(kbdev); - else if (kbdev->pm.backend.callback_power_off) + if (kbdev->pm.backend.callback_power_off) kbdev->pm.backend.callback_power_off(kbdev); return true; } @@ -1715,32 +1732,10 @@ static void kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id) kbdev->hw_quirks_sc = kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG)); - /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. - * See PRLAM-8443 and needed due to MIDGLES-3539. - * See PRLAM-11035. - */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) || - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035)) - kbdev->hw_quirks_sc |= SC_LS_PAUSEBUFFER_DISABLE; - - /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. - * See PRLAM-10327. - */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327)) - kbdev->hw_quirks_sc |= SC_SDC_DISABLE_OQ_DISCARD; - - /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. - * See PRLAM-10797. - */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797)) - kbdev->hw_quirks_sc |= SC_ENABLE_TEXGRD_FLAGS; - - if (!kbase_hw_has_issue(kbdev, GPUCORE_1619)) { - if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ - kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; - else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */ - kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; - } + if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ + kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE; + else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */ + kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES; if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_2968_TTRX_3162)) kbdev->hw_quirks_sc |= SC_VAR_ALGORITHM; @@ -1864,13 +1859,22 @@ static void reenable_protected_mode_hwcnt(struct kbase_device *kbdev) static int kbase_pm_do_reset(struct kbase_device *kbdev) { struct kbasep_reset_timeout_data rtdata; + int ret; KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0); KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev); - kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), - GPU_COMMAND_SOFT_RESET); + if (kbdev->pm.backend.callback_soft_reset) { + ret = kbdev->pm.backend.callback_soft_reset(kbdev); + if (ret < 0) + return ret; + else if (ret > 0) + return 0; + } else { + kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), + GPU_COMMAND_SOFT_RESET); + } /* Unmask the reset complete interrupt only */ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h index 4d4b2218abd9..6cab4535442d 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h @@ -142,13 +142,12 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume); * * @kbdev: The kbase device structure for the device (must be a valid * pointer) - * @is_suspend: true if clock off due to suspend, false otherwise * * Return: true if clock was turned off, or * false if clock can not be turned off due to pending page/bus fault * workers. Caller must flush MMU workqueues and retry */ -bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend); +bool kbase_pm_clock_off(struct kbase_device *kbdev); /** * kbase_pm_enable_interrupts - Enable interrupts on the device. @@ -244,10 +243,24 @@ void kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); */ void kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev); +/** + * kbase_pm_update_dynamic_cores_onoff - Update the L2 and shader power state + * machines after changing shader core + * availability + * + * It can be called in any status, so need to check the l2 and shader core + * power status in this function or it will break shader/l2 state machine + * + * Caller must hold hwaccess_lock + * + * @kbdev: The kbase device structure for the device (must be a valid pointer) + */ +void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev); + /** * kbase_pm_update_cores_state_nolock - Variant of kbase_pm_update_cores_state() * where the caller must hold - * kbase_device.pm.power_change_lock + * kbase_device.hwaccess_lock * * @kbdev: The kbase device structure for the device (must be a valid pointer) */ @@ -487,10 +500,8 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume); * * @kbdev: The kbase device structure for the device (must be a valid * pointer) - * @is_suspend: true if power off due to suspend, - * false otherwise */ -void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend); +void kbase_pm_do_poweroff(struct kbase_device *kbdev); #if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, @@ -661,11 +672,14 @@ void kbase_pm_protected_entry_override_disable(struct kbase_device *kbdev); */ extern bool corestack_driver_control; -/* If true, disable powering-down of individual cores, and just power-down at - * the top-level using platform-specific code. - * If false, use the expected behaviour of controlling the individual cores - * from within the driver. +/** + * kbase_pm_is_l2_desired - Check whether l2 is desired + * + * @kbdev: Device pointer + * + * This shall be called to check whether l2 is needed to power on + * + * Return: true if l2 need to power on */ -extern bool platform_power_down_only; - +bool kbase_pm_is_l2_desired(struct kbase_device *kbdev); #endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c index 31a92f87f019..f0e2d22879d3 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c @@ -45,33 +45,10 @@ static const struct kbase_pm_policy *const all_policy_list[] = { #endif /* CONFIG_MALI_BIFROST_NO_MALI */ }; -static void generate_filtered_policy_list(struct kbase_device *kbdev) +void kbase_pm_policy_init(struct kbase_device *kbdev) { - size_t i; - - for (i = 0; i < ARRAY_SIZE(all_policy_list); ++i) { - const struct kbase_pm_policy *pol = all_policy_list[i]; - - BUILD_BUG_ON(ARRAY_SIZE(all_policy_list) > - KBASE_PM_MAX_NUM_POLICIES); - if (platform_power_down_only && - (pol->flags & KBASE_PM_POLICY_FLAG_DISABLED_WITH_POWER_DOWN_ONLY)) - continue; - - kbdev->policy_list[kbdev->policy_count++] = pol; - } -} - -int kbase_pm_policy_init(struct kbase_device *kbdev) -{ - generate_filtered_policy_list(kbdev); - if (kbdev->policy_count == 0) - return -EINVAL; - - kbdev->pm.backend.pm_current_policy = kbdev->policy_list[0]; + kbdev->pm.backend.pm_current_policy = all_policy_list[0]; kbdev->pm.backend.pm_current_policy->init(kbdev); - - return 0; } void kbase_pm_policy_term(struct kbase_device *kbdev) @@ -106,7 +83,7 @@ void kbase_pm_update_active(struct kbase_device *kbdev) pm->backend.poweron_required = true; spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } else { - /* Cancel the the invocation of + /* Cancel the invocation of * kbase_pm_gpu_poweroff_wait_wq() from the L2 state * machine. This is safe - it * invoke_poweroff_wait_wq_when_l2_off is true, then @@ -130,13 +107,37 @@ void kbase_pm_update_active(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); /* Power off the GPU immediately */ - kbase_pm_do_poweroff(kbdev, false); + kbase_pm_do_poweroff(kbdev); } else { spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } } } +void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev) +{ + bool shaders_desired; + + lockdep_assert_held(&kbdev->hwaccess_lock); + lockdep_assert_held(&kbdev->pm.lock); + + if (kbdev->pm.backend.pm_current_policy == NULL) + return; + if (kbdev->pm.backend.poweroff_wait_in_progress) + return; + /* In protected transition, don't allow outside shader core request + * affect transition, return directly + */ + if (kbdev->pm.backend.protected_transition_override) + return; + + shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); + + if (shaders_desired && kbase_pm_is_l2_desired(kbdev)) { + kbase_pm_update_state(kbdev); + } +} + void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) { bool shaders_desired; @@ -178,11 +179,10 @@ void kbase_pm_update_cores_state(struct kbase_device *kbdev) int kbase_pm_list_policies(struct kbase_device *kbdev, const struct kbase_pm_policy * const **list) { - WARN_ON(kbdev->policy_count == 0); if (list) - *list = kbdev->policy_list; + *list = all_policy_list; - return kbdev->policy_count; + return ARRAY_SIZE(all_policy_list); } KBASE_EXPORT_TEST_API(kbase_pm_list_policies); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h index 966fce778e44..f103ef0c01e4 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2015, 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -33,11 +33,8 @@ * @kbdev: The kbase device structure for the device (must be a valid pointer) * * Must be called before calling any other policy function - * - * Return: 0 if the power policy framework was successfully - * initialized, -errno otherwise. */ -int kbase_pm_policy_init(struct kbase_device *kbdev); +void kbase_pm_policy_init(struct kbase_device *kbdev); /** * kbase_pm_policy_term - Terminate power policy framework diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h index 3f89eb5b6e87..2f573de5ab3a 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h @@ -33,6 +33,7 @@ KBASEP_SHADER_STATE(PEND_ON_CORESTACK_ON) KBASEP_SHADER_STATE(ON_CORESTACK_ON) KBASEP_SHADER_STATE(ON_CORESTACK_ON_RECHECK) KBASEP_SHADER_STATE(WAIT_OFF_CORESTACK_ON) +KBASEP_SHADER_STATE(WAIT_GPU_IDLE) KBASEP_SHADER_STATE(WAIT_FINISHED_CORESTACK_ON) KBASEP_SHADER_STATE(L2_FLUSHING_CORESTACK_ON) KBASEP_SHADER_STATE(READY_OFF_CORESTACK_ON) diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c index 1cb1a943af9c..057bf10082eb 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c @@ -64,48 +64,3 @@ void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, kbase_pm_release_gpu_cycle_counter(kbdev); } - -/** - * kbase_wait_write_flush - Wait for GPU write flush - * @kbdev: Kbase device - * - * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush - * its write buffer. - * - * Only in use for BASE_HW_ISSUE_6367 - * - * Note : If GPU resets occur then the counters are reset to zero, the delay may - * not be as expected. - */ -#ifndef CONFIG_MALI_BIFROST_NO_MALI -void kbase_wait_write_flush(struct kbase_device *kbdev) -{ - u32 base_count = 0; - - /* - * The caller must be holding onto the kctx or the call is from - * userspace. - */ - kbase_pm_context_active(kbdev); - kbase_pm_request_gpu_cycle_counter(kbdev); - - while (true) { - u32 new_count; - - new_count = kbase_reg_read(kbdev, - GPU_CONTROL_REG(CYCLE_COUNT_LO)); - /* First time around, just store the count. */ - if (base_count == 0) { - base_count = new_count; - continue; - } - - /* No need to handle wrapping, unsigned maths works for this. */ - if ((new_count - base_count) > 1000) - break; - } - - kbase_pm_release_gpu_cycle_counter(kbdev); - kbase_pm_context_idle(kbdev); -} -#endif /* CONFIG_MALI_BIFROST_NO_MALI */ diff --git a/drivers/gpu/arm/bifrost/build.bp b/drivers/gpu/arm/bifrost/build.bp index 53acc74c3ac2..64eeed295e3c 100644 --- a/drivers/gpu/arm/bifrost/build.bp +++ b/drivers/gpu/arm/bifrost/build.bp @@ -102,9 +102,6 @@ bob_kernel_module { mali_corestack: { kbuild_options: ["CONFIG_MALI_CORESTACK=y"], }, - mali_platform_power_down_only: { - kbuild_options: ["CONFIG_MALI_PLATFORM_POWER_DOWN_ONLY=y"], - }, mali_error_inject: { kbuild_options: ["CONFIG_MALI_BIFROST_ERROR_INJECT=y"], }, diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c index a6c70099b13c..0386b066da03 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c @@ -121,7 +121,8 @@ const char *kbase_ipa_model_name_from_id(u32 gpu_id) } KBASE_EXPORT_TEST_API(kbase_ipa_model_name_from_id); -static struct device_node *get_model_dt_node(struct kbase_ipa_model *model) +static struct device_node *get_model_dt_node(struct kbase_ipa_model *model, + bool dt_required) { struct device_node *model_dt_node; char compat_string[64]; @@ -136,9 +137,10 @@ static struct device_node *get_model_dt_node(struct kbase_ipa_model *model) model_dt_node = of_find_compatible_node(model->kbdev->dev->of_node, NULL, compat_string); if (!model_dt_node && !model->missing_dt_node_warning) { - dev_warn(model->kbdev->dev, - "Couldn't find power_model DT node matching \'%s\'\n", - compat_string); + if (dt_required) + dev_warn(model->kbdev->dev, + "Couldn't find power_model DT node matching \'%s\'\n", + compat_string); model->missing_dt_node_warning = true; } @@ -150,7 +152,8 @@ int kbase_ipa_model_add_param_s32(struct kbase_ipa_model *model, size_t num_elems, bool dt_required) { int err, i; - struct device_node *model_dt_node = get_model_dt_node(model); + struct device_node *model_dt_node = get_model_dt_node(model, + dt_required); char *origin; err = of_property_read_u32_array(model_dt_node, name, addr, num_elems); @@ -199,7 +202,8 @@ int kbase_ipa_model_add_param_string(struct kbase_ipa_model *model, size_t size, bool dt_required) { int err; - struct device_node *model_dt_node = get_model_dt_node(model); + struct device_node *model_dt_node = get_model_dt_node(model, + dt_required); const char *string_prop_value; char *origin; diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h index 3d249722fbf0..c25f870097b7 100644 --- a/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h +++ b/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h @@ -40,17 +40,13 @@ enum base_hw_feature { BASE_HW_FEATURE_MSAA_16X, BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, - BASE_HW_FEATURE_OPTIMIZED_COVERAGE_MASK, BASE_HW_FEATURE_T7XX_PAIRING_RULES, BASE_HW_FEATURE_LD_ST_LEA_TEX, BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, - BASE_HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4, - BASE_HW_FEATURE_IMAGES_IN_FRAGMENT_SHADERS, BASE_HW_FEATURE_TEST4_DATUM_MODE, BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, BASE_HW_FEATURE_BRNDOUT_KILL, BASE_HW_FEATURE_WARPING, - BASE_HW_FEATURE_V4, BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_MODE, BASE_HW_FEATURE_COHERENCY_REG, diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h index 7448608ba718..1a28bb1ea982 100644 --- a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h +++ b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h @@ -30,87 +30,10 @@ enum base_hw_issue { BASE_HW_ISSUE_5736, - BASE_HW_ISSUE_6367, - BASE_HW_ISSUE_6398, - BASE_HW_ISSUE_6402, - BASE_HW_ISSUE_6787, - BASE_HW_ISSUE_7027, - BASE_HW_ISSUE_7144, - BASE_HW_ISSUE_7304, - BASE_HW_ISSUE_8073, - BASE_HW_ISSUE_8186, - BASE_HW_ISSUE_8215, - BASE_HW_ISSUE_8245, - BASE_HW_ISSUE_8250, - BASE_HW_ISSUE_8260, - BASE_HW_ISSUE_8280, - BASE_HW_ISSUE_8316, - BASE_HW_ISSUE_8381, - BASE_HW_ISSUE_8394, - BASE_HW_ISSUE_8401, - BASE_HW_ISSUE_8408, - BASE_HW_ISSUE_8443, - BASE_HW_ISSUE_8456, - BASE_HW_ISSUE_8564, - BASE_HW_ISSUE_8634, - BASE_HW_ISSUE_8778, - BASE_HW_ISSUE_8791, - BASE_HW_ISSUE_8833, - BASE_HW_ISSUE_8879, - BASE_HW_ISSUE_8896, - BASE_HW_ISSUE_8975, - BASE_HW_ISSUE_8986, - BASE_HW_ISSUE_8987, - BASE_HW_ISSUE_9010, - BASE_HW_ISSUE_9418, - BASE_HW_ISSUE_9423, BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_9510, - BASE_HW_ISSUE_9566, - BASE_HW_ISSUE_9630, - BASE_HW_ISSUE_10127, - BASE_HW_ISSUE_10327, - BASE_HW_ISSUE_10410, - BASE_HW_ISSUE_10471, - BASE_HW_ISSUE_10472, - BASE_HW_ISSUE_10487, - BASE_HW_ISSUE_10607, - BASE_HW_ISSUE_10632, - BASE_HW_ISSUE_10649, - BASE_HW_ISSUE_10676, BASE_HW_ISSUE_10682, - BASE_HW_ISSUE_10684, - BASE_HW_ISSUE_10797, - BASE_HW_ISSUE_10817, - BASE_HW_ISSUE_10821, - BASE_HW_ISSUE_10883, - BASE_HW_ISSUE_10931, - BASE_HW_ISSUE_10946, - BASE_HW_ISSUE_10959, - BASE_HW_ISSUE_10969, - BASE_HW_ISSUE_10984, - BASE_HW_ISSUE_10995, - BASE_HW_ISSUE_11012, - BASE_HW_ISSUE_11035, - BASE_HW_ISSUE_11042, - BASE_HW_ISSUE_11051, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_11056, - BASE_HW_ISSUE_T720_1386, - BASE_HW_ISSUE_T76X_26, - BASE_HW_ISSUE_T76X_1909, - BASE_HW_ISSUE_T76X_1963, - BASE_HW_ISSUE_T76X_3086, - BASE_HW_ISSUE_T76X_3542, - BASE_HW_ISSUE_T76X_3556, - BASE_HW_ISSUE_T76X_3700, - BASE_HW_ISSUE_T76X_3793, BASE_HW_ISSUE_T76X_3953, - BASE_HW_ISSUE_T76X_3960, - BASE_HW_ISSUE_T76X_3964, - BASE_HW_ISSUE_T76X_3966, - BASE_HW_ISSUE_T76X_3979, - BASE_HW_ISSUE_T83X_817, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, @@ -120,7 +43,6 @@ enum base_hw_issue { BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, - GPUCORE_1619, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TMIX_8438, diff --git a/drivers/gpu/arm/bifrost/mali_base_kernel.h b/drivers/gpu/arm/bifrost/mali_base_kernel.h index a8ab408a1ded..bd42e5c63a87 100644 --- a/drivers/gpu/arm/bifrost/mali_base_kernel.h +++ b/drivers/gpu/arm/bifrost/mali_base_kernel.h @@ -1782,11 +1782,17 @@ static inline int base_context_mmu_group_id_get( #define BASE_TIMEINFO_TIMESTAMP_FLAG (1UL << 1) /* For GPU cycle counter */ #define BASE_TIMEINFO_CYCLE_COUNTER_FLAG (1UL << 2) +/* Specify kernel GPU register timestamp */ +#define BASE_TIMEINFO_KERNEL_SOURCE_FLAG (1UL << 30) +/* Specify userspace cntvct_el0 timestamp source */ +#define BASE_TIMEINFO_USER_SOURCE_FLAG (1UL << 31) #define BASE_TIMEREQUEST_ALLOWED_FLAGS (\ BASE_TIMEINFO_MONOTONIC_FLAG | \ BASE_TIMEINFO_TIMESTAMP_FLAG | \ - BASE_TIMEINFO_CYCLE_COUNTER_FLAG) + BASE_TIMEINFO_CYCLE_COUNTER_FLAG | \ + BASE_TIMEINFO_KERNEL_SOURCE_FLAG | \ + BASE_TIMEINFO_USER_SOURCE_FLAG) #endif /* _BASE_KERNEL_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config.h b/drivers/gpu/arm/bifrost/mali_kbase_config.h index 1637fcbc4d29..69723eaf188e 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_config.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_config.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2017, 2019 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -209,6 +209,18 @@ struct kbase_pm_callback_conf { * suspeneded by runtime PM, else OS error code */ int (*power_runtime_idle_callback)(struct kbase_device *kbdev); + + /* + * Optional callback for software reset + * + * This callback will be called by the power management core to trigger + * a GPU soft reset. + * + * Return 0 if the soft reset was successful and the RESET_COMPLETED + * interrupt will be raised, or a positive value if the interrupt won't + * be raised. On error, return the corresponding OS error code. + */ + int (*soft_reset_callback)(struct kbase_device *kbdev); }; #ifdef CONFIG_OF diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h index 447e05904676..6a1083c0bd76 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h @@ -150,7 +150,6 @@ enum { * Default minimum number of scheduling ticks before jobs are hard-stopped */ #define DEFAULT_JS_HARD_STOP_TICKS_SS (50) /* 5s */ -#define DEFAULT_JS_HARD_STOP_TICKS_SS_8408 (300) /* 30s */ /** * Default minimum number of scheduling ticks before CL jobs are hard-stopped. @@ -174,7 +173,6 @@ enum { * "stuck" job */ #define DEFAULT_JS_RESET_TICKS_SS (55) /* 5.5s */ -#define DEFAULT_JS_RESET_TICKS_SS_8408 (450) /* 45s */ /** * Default minimum number of scheduling ticks before the GPU is reset to clear a diff --git a/drivers/gpu/arm/bifrost/mali_kbase_context.c b/drivers/gpu/arm/bifrost/mali_kbase_context.c index e72ce7084f7b..1cd854e5d7cf 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_context.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_context.c @@ -45,6 +45,7 @@ kbase_create_context(struct kbase_device *kbdev, bool is_compat, struct page *p; struct kbasep_js_kctx_info *js_kctx_info = NULL; unsigned long irq_flags = 0; + const unsigned long cookies_mask = KBASE_COOKIE_MASK; if (WARN_ON(!kbdev)) goto out; @@ -103,8 +104,6 @@ kbase_create_context(struct kbase_device *kbdev, bool is_compat, if (err) goto free_jd; - atomic_set(&kctx->drain_pending, 0); - mutex_init(&kctx->reg_lock); spin_lock_init(&kctx->mem_partials_lock); @@ -129,7 +128,7 @@ kbase_create_context(struct kbase_device *kbdev, bool is_compat, init_waitqueue_head(&kctx->event_queue); - kctx->cookies = KBASE_COOKIE_MASK; + bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG); /* Make sure page 0 is not used... */ err = kbase_region_tracker_init(kctx); @@ -287,16 +286,19 @@ void kbase_destroy_context(struct kbase_context *kctx) p, false); /* free pending region setups */ - pending_regions_to_clean = (~kctx->cookies) & KBASE_COOKIE_MASK; + pending_regions_to_clean = KBASE_COOKIE_MASK; + bitmap_andnot(&pending_regions_to_clean, &pending_regions_to_clean, + kctx->cookies, BITS_PER_LONG); while (pending_regions_to_clean) { - unsigned int cookie = __ffs(pending_regions_to_clean); + unsigned int cookie = find_first_bit(&pending_regions_to_clean, + BITS_PER_LONG); BUG_ON(!kctx->pending_regions[cookie]); kbase_reg_pending_dtor(kbdev, kctx->pending_regions[cookie]); kctx->pending_regions[cookie] = NULL; - pending_regions_to_clean &= ~(1UL << cookie); + bitmap_clear(&pending_regions_to_clean, cookie, 1); } kbase_region_tracker_term(kctx); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c index 073e16309dc9..9ec53dc500d3 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c @@ -1240,7 +1240,7 @@ static int kbase_api_sticky_resource_map(struct kbase_context *kctx, if (ret != 0) { while (i > 0) { i--; - kbase_sticky_resource_release(kctx, NULL, gpu_addr[i]); + kbase_sticky_resource_release_force(kctx, NULL, gpu_addr[i]); } } @@ -1268,7 +1268,7 @@ static int kbase_api_sticky_resource_unmap(struct kbase_context *kctx, kbase_gpu_vm_lock(kctx); for (i = 0; i < unmap->count; i++) { - if (!kbase_sticky_resource_release(kctx, NULL, gpu_addr[i])) { + if (!kbase_sticky_resource_release_force(kctx, NULL, gpu_addr[i])) { /* Invalid resource, but we keep going anyway */ ret = -EINVAL; } @@ -1922,6 +1922,7 @@ static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, if (items == 1) new_core_mask[1] = new_core_mask[2] = new_core_mask[0]; + mutex_lock(&kbdev->pm.lock); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); shader_present = kbdev->gpu_props.props.raw_props.shader_present; @@ -1962,6 +1963,7 @@ static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, unlock: spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + mutex_unlock(&kbdev->pm.lock); end: return err; } @@ -2122,8 +2124,6 @@ static ssize_t set_js_timeouts(struct device *dev, struct device_attribute *attr UPDATE_TIMEOUT(soft_stop_ticks_cl, js_soft_stop_ms_cl, DEFAULT_JS_SOFT_STOP_TICKS_CL); UPDATE_TIMEOUT(hard_stop_ticks_ss, js_hard_stop_ms_ss, - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? - DEFAULT_JS_HARD_STOP_TICKS_SS_8408 : DEFAULT_JS_HARD_STOP_TICKS_SS); UPDATE_TIMEOUT(hard_stop_ticks_cl, js_hard_stop_ms_cl, DEFAULT_JS_HARD_STOP_TICKS_CL); @@ -2131,8 +2131,6 @@ static ssize_t set_js_timeouts(struct device *dev, struct device_attribute *attr js_hard_stop_ms_dumping, DEFAULT_JS_HARD_STOP_TICKS_DUMPING); UPDATE_TIMEOUT(gpu_reset_ticks_ss, js_reset_ms_ss, - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408) ? - DEFAULT_JS_RESET_TICKS_SS_8408 : DEFAULT_JS_RESET_TICKS_SS); UPDATE_TIMEOUT(gpu_reset_ticks_cl, js_reset_ms_cl, DEFAULT_JS_RESET_TICKS_CL); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c index bda05602de5e..35853a3a26b5 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2017-2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2019 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -32,16 +32,6 @@ int kbase_ctx_sched_init(struct kbase_device *kbdev) /* These two must be recalculated if nr_hw_address_spaces changes * (e.g. for HW workarounds) */ kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) { - bool use_workaround; - - use_workaround = DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE; - if (use_workaround) { - dev_dbg(kbdev->dev, "GPU has HW ISSUE 8987, and driver configured for security workaround: 1 address space only"); - kbdev->nr_user_address_spaces = 1; - } - } - kbdev->as_free = as_present; /* All ASs initially free */ memset(kbdev->as_to_kctx, 0, sizeof(kbdev->as_to_kctx)); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_defs.h index 7e36a50a492a..0a3b9514247e 100755 --- a/drivers/gpu/arm/bifrost/mali_kbase_defs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_defs.h @@ -156,7 +156,7 @@ #define KBASEP_AS_NR_INVALID (-1) #define KBASE_LOCK_REGION_MAX_SIZE (63) -#define KBASE_LOCK_REGION_MIN_SIZE (11) +#define KBASE_LOCK_REGION_MIN_SIZE (15) #define KBASE_TRACE_SIZE_LOG2 8 /* 256 entries */ #define KBASE_TRACE_SIZE (1 << KBASE_TRACE_SIZE_LOG2) @@ -543,8 +543,6 @@ struct kbase_ext_res { * on the GPU. * @sched_priority: Priority of the atom for Job scheduling, as per the * KBASE_JS_ATOM_SCHED_PRIO_*. - * @poking: Indicates whether poking of MMU is ongoing for the atom, - * as a WA for the issue HW_ISSUE_8316. * @completed: Wait queue to wait upon for the completion of atom. * @status: Indicates at high level at what stage the atom is in, * as per KBASE_JD_ATOM_STATE_*, that whether it is not in @@ -697,8 +695,6 @@ struct kbase_jd_atom { u32 ticks; int sched_priority; - int poking; - wait_queue_head_t completed; enum kbase_jd_atom_state status; #ifdef CONFIG_GPU_TRACEPOINTS @@ -853,15 +849,6 @@ struct kbase_device_info { u32 features; }; -/** Poking state for BASE_HW_ISSUE_8316 */ -enum { - KBASE_AS_POKE_STATE_IN_FLIGHT = 1<<0, - KBASE_AS_POKE_STATE_KILLING_POKE = 1<<1 -}; - -/** Poking state for BASE_HW_ISSUE_8316 */ -typedef u32 kbase_as_poke_state; - struct kbase_mmu_setup { u64 transtab; u64 memattr; @@ -895,15 +882,6 @@ struct kbase_fault { * @pf_data: Data relating to page fault. * @bf_data: Data relating to bus fault. * @current_setup: Stores the MMU configuration for this address space. - * @poke_wq: Workqueue to process the work items queue for poking the - * MMU as a WA for BASE_HW_ISSUE_8316. - * @poke_work: Work item to do the poking of MMU for this address space. - * @poke_refcount: Refcount for the need of poking MMU. While the refcount is - * non zero the poking of MMU will continue. - * Protected by hwaccess_lock. - * @poke_state: State indicating whether poking is in progress or it has - * been stopped. Protected by hwaccess_lock. - * @poke_timer: Timer used to schedule the poking at regular intervals. */ struct kbase_as { int number; @@ -913,11 +891,6 @@ struct kbase_as { struct kbase_fault pf_data; struct kbase_fault bf_data; struct kbase_mmu_setup current_setup; - struct workqueue_struct *poke_wq; - struct work_struct poke_work; - int poke_refcount; - kbase_as_poke_state poke_state; - struct hrtimer poke_timer; }; /** @@ -1505,8 +1478,6 @@ struct kbase_devfreq_queue_info { * Job Scheduler * @l2_size_override: Used to set L2 cache size via device tree blob * @l2_hash_override: Used to set L2 cache hash via device tree blob - * @policy_list: A filtered list of policies available in the system. - * @policy_count: Number of policies in the @policy_list. */ struct kbase_device { u32 hw_quirks_sc; @@ -1747,9 +1718,6 @@ struct kbase_device { /* See KBASE_JS_*_PRIORITY_MODE for details. */ u32 js_ctx_scheduling_mode; - - const struct kbase_pm_policy *policy_list[KBASE_PM_MAX_NUM_POLICIES]; - int policy_count; }; /** @@ -2110,9 +2078,6 @@ struct kbase_sub_alloc { * @ext_res_meta_head: A list of sticky external resources which were requested to * be mapped on GPU side, through a softjob atom of type * EXT_RES_MAP or STICKY_RESOURCE_MAP ioctl. - * @drain_pending: Used to record that a flush/invalidate of the GPU caches was - * requested from atomic context, so that the next flush request - * can wait for the flush of GPU writes. * @age_count: Counter incremented on every call to jd_submit_atom, * atom is assigned the snapshot of this counter, which * is used to determine the atom's age when it is added to @@ -2163,7 +2128,7 @@ struct kbase_context { struct rb_root reg_rbtree_exec; - unsigned long cookies; + DECLARE_BITMAP(cookies, BITS_PER_LONG); struct kbase_va_region *pending_regions[BITS_PER_LONG]; wait_queue_head_t event_queue; @@ -2251,8 +2216,6 @@ struct kbase_context { struct list_head ext_res_meta_head; - atomic_t drain_pending; - u32 age_count; u8 trim_level; @@ -2301,6 +2264,7 @@ struct kbasep_gwt_list_element { * which is mapped. * @gpu_addr: The GPU virtual address the resource is * mapped to. + * @ref: Reference count. * * External resources can be mapped into multiple contexts as well as the same * context multiple times. @@ -2316,6 +2280,7 @@ struct kbase_ctx_ext_res_meta { struct list_head ext_res_node; struct kbase_mem_phy_alloc *alloc; u64 gpu_addr; + u32 ref; }; enum kbase_reg_access_type { diff --git a/drivers/gpu/arm/bifrost/mali_kbase_device.c b/drivers/gpu/arm/bifrost/mali_kbase_device.c index a265082c7ee8..89db1746c5e5 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_device.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_device.c @@ -79,34 +79,12 @@ static int kbase_device_as_init(struct kbase_device *kbdev, int i) INIT_WORK(&kbdev->as[i].work_pagefault, page_fault_worker); INIT_WORK(&kbdev->as[i].work_busfault, bus_fault_worker); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) { - struct hrtimer *poke_timer = &kbdev->as[i].poke_timer; - struct work_struct *poke_work = &kbdev->as[i].poke_work; - - kbdev->as[i].poke_wq = - alloc_workqueue("mali_mmu%d_poker", 0, 1, i); - if (!kbdev->as[i].poke_wq) { - destroy_workqueue(kbdev->as[i].pf_wq); - return -EINVAL; - } - INIT_WORK(poke_work, kbasep_as_do_poke); - - hrtimer_init(poke_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - - poke_timer->function = kbasep_as_poke_timer_callback; - - kbdev->as[i].poke_refcount = 0; - kbdev->as[i].poke_state = 0u; - } - return 0; } static void kbase_device_as_term(struct kbase_device *kbdev, int i) { destroy_workqueue(kbdev->as[i].pf_wq); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) - destroy_workqueue(kbdev->as[i].poke_wq); } static int kbase_device_all_as_init(struct kbase_device *kbdev) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h index c3b60e6ea2ec..cfda5c4b8129 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h @@ -290,6 +290,15 @@ u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev); void kbase_job_slot_hardstop(struct kbase_context *kctx, int js, struct kbase_jd_atom *target_katom); +/** + * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms + * currently on the GPU + * @kbdev: Device pointer + * + * Return: true if there are any atoms on the GPU, false otherwise + */ +bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev); + /* Object containing callbacks for enabling/disabling protected mode, used * on GPU which supports protected mode switching natively. */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h index cbe6ec1f48ad..ca0cd797e5a2 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_time.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014,2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014,2018-2019 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -39,24 +39,4 @@ void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, u64 *system_time, struct timespec *ts); -/** - * kbase_wait_write_flush() - Wait for GPU write flush - * @kbdev: Kbase device - * - * Wait 1000 GPU clock cycles. This delay is known to give the GPU time to flush - * its write buffer. - * - * If GPU resets occur then the counters are reset to zero, the delay may not be - * as expected. - * - * This function is only in use for BASE_HW_ISSUE_6367 - */ -#ifdef CONFIG_MALI_BIFROST_NO_MALI -static inline void kbase_wait_write_flush(struct kbase_device *kbdev) -{ -} -#else -void kbase_wait_write_flush(struct kbase_device *kbdev); -#endif - #endif /* _KBASE_BACKEND_TIME_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c index 459396799ec1..73d1802c70bf 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwcnt_gpu.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2019 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -296,11 +296,7 @@ int kbase_hwcnt_gpu_info_init( info->v5.l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS; info->v5.core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1; #else - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) { - info->type = KBASE_HWCNT_GPU_GROUP_TYPE_V4; - info->v4.cg_count = kbdev->gpu_props.num_core_groups; - info->v4.cgs = kbdev->gpu_props.props.coherency_info.group; - } else { + { const struct base_gpu_props *props = &kbdev->gpu_props.props; const size_t l2_count = props->l2_props.num_l2_slices; const size_t core_mask = diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h b/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h index 9b138e555ff5..9694d620fada 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h @@ -82,7 +82,6 @@ extern "C" { * 11.17: * - Added BASE_JD_REQ_JOB_SLOT. * - Reused padding field in base_jd_atom_v2 to pass job slot number. - * 11.18: * - New ioctl: KBASE_IOCTL_GET_CPU_GPU_TIMEINFO */ #define BASE_UK_VERSION_MAJOR 11 @@ -741,6 +740,7 @@ union kbase_ioctl_get_cpu_gpu_timeinfo { #define KBASE_IOCTL_GET_CPU_GPU_TIMEINFO \ _IOWR(KBASE_IOCTL_TYPE, 50, union kbase_ioctl_get_cpu_gpu_timeinfo) + /*************** * test ioctls * ***************/ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd.c b/drivers/gpu/arm/bifrost/mali_kbase_jd.c index be9a51d55b8e..77cfa2fc3699 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_jd.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_jd.c @@ -546,7 +546,6 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, struct list_head *completed_jobs_ctx) { struct kbase_context *kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; struct list_head completed_jobs; struct list_head runnable_jobs; bool need_to_try_schedule_context = false; @@ -566,22 +565,6 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, } } - /* With PRLAM-10817 or PRLAM-10959 the last tile of a fragment job being soft-stopped can fail with - * BASE_JD_EVENT_TILE_RANGE_FAULT. - * - * So here if the fragment job failed with TILE_RANGE_FAULT and it has been soft-stopped, then we promote the - * error code to BASE_JD_EVENT_DONE - */ - - if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10817) || kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10959)) && - katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT) { - if ((katom->core_req & BASE_JD_REQ_FS) && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED)) { - /* Promote the failure to job done */ - katom->event_code = BASE_JD_EVENT_DONE; - katom->atom_flags = katom->atom_flags & (~KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED); - } - } - katom->status = KBASE_JD_ATOM_STATE_COMPLETED; list_add_tail(&katom->jd_item, &completed_jobs); @@ -1193,9 +1176,6 @@ void kbase_jd_done_worker(struct work_struct *data) (unsigned long)katom->event_code, katom->slot_nr); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) - kbase_as_poking_timer_release_atom(kbdev, kctx, katom); - /* Retain state before the katom disappears */ kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c index cf20a35a5c98..fc6baebec8d4 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_jd_debugfs.c @@ -46,18 +46,16 @@ static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, switch (atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) { case BASE_JD_REQ_SOFT_FENCE_TRIGGER: res = kbase_sync_fence_out_info_get(atom, &info); - if (0 == res) { + if (res == 0) seq_printf(sfile, "Sa([%p]%d) ", info.fence, info.status); - break; - } + break; case BASE_JD_REQ_SOFT_FENCE_WAIT: res = kbase_sync_fence_in_info_get(atom, &info); - if (0 == res) { + if (res == 0) seq_printf(sfile, "Wa([%p]%d) ", info.fence, info.status); - break; - } + break; default: break; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js.c b/drivers/gpu/arm/bifrost/mali_kbase_js.c index 4d9c14172d23..e0bac144a002 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_js.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_js.c @@ -451,16 +451,10 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) jsdd->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; jsdd->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; jsdd->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) - jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS_8408; - else - jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; + jsdd->hard_stop_ticks_ss = DEFAULT_JS_HARD_STOP_TICKS_SS; jsdd->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; jsdd->hard_stop_ticks_dumping = DEFAULT_JS_HARD_STOP_TICKS_DUMPING; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) - jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS_8408; - else - jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; + jsdd->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; jsdd->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; jsdd->gpu_reset_ticks_dumping = DEFAULT_JS_RESET_TICKS_DUMPING; jsdd->ctx_timeslice_ns = DEFAULT_JS_CTX_TIMESLICE_NS; @@ -1016,14 +1010,7 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, ret = false; break; } - /* Cross-slot dependencies must not violate - * PRLAM-8987 affinity restrictions */ - if (kbase_hw_has_issue(kbdev, - BASE_HW_ISSUE_8987) && - (js == 2 || dep_js == 2)) { - ret = false; - break; - } + has_x_dep = true; } @@ -2026,11 +2013,6 @@ bool kbase_js_is_atom_valid(struct kbase_device *kbdev, BASE_JD_REQ_T))) return false; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987) && - (katom->core_req & BASE_JD_REQ_ONLY_COMPUTE) && - (katom->core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_T))) - return false; - if ((katom->core_req & BASE_JD_REQ_JOB_SLOT) && (katom->jobslot >= BASE_JM_MAX_NR_SLOTS)) return false; @@ -2051,8 +2033,6 @@ static int kbase_js_get_slot(struct kbase_device *kbdev, if (katom->device_nr == 1 && kbdev->gpu_props.num_core_groups == 2) return 2; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) - return 2; } return 1; @@ -2263,9 +2243,6 @@ static void js_return_worker(struct work_struct *data) kbase_backend_complete_wq(kbdev, katom); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) - kbase_as_poking_timer_release_atom(kbdev, kctx, katom); - kbasep_js_atom_retained_state_copy(&retained_state, katom); mutex_lock(&js_devdata->queue_mutex); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.c b/drivers/gpu/arm/bifrost/mali_kbase_mem.c index fa05f3492c25..45899833e23f 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.c @@ -970,6 +970,9 @@ int kbase_mem_init(struct kbase_device *kbdev) "Failed to get memory group manger module\n"); err = -ENODEV; kbdev->mgm_dev = NULL; + } else { + dev_info(kbdev->dev, + "Memory group manager successfully loaded\n"); } } of_node_put(mgm_node); @@ -1154,7 +1157,7 @@ void kbase_free_alloced_region(struct kbase_va_region *reg) * Remove the region from the sticky resource metadata * list should it be there. */ - kbase_sticky_resource_release(kctx, NULL, + kbase_sticky_resource_release_force(kctx, NULL, reg->start_pfn << PAGE_SHIFT); kbase_mem_phy_alloc_put(reg->cpu_alloc); @@ -1715,7 +1718,7 @@ int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) /* ask to unlink the cookie as we'll free it */ kctx->pending_regions[cookie] = NULL; - kctx->cookies |= (1UL << cookie); + bitmap_set(kctx->cookies, cookie, 1); kbase_free_alloced_region(reg); } else { @@ -3795,6 +3798,7 @@ struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) { if (walker->gpu_addr == gpu_addr) { meta = walker; + meta->ref++; break; } } @@ -3819,6 +3823,7 @@ struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( * for the physical resource. */ meta->alloc = kbase_map_external_resource(kctx, reg, NULL); + meta->ref = 1; if (!meta->alloc) goto fail_map; @@ -3836,32 +3841,28 @@ failed: return NULL; } -bool kbase_sticky_resource_release(struct kbase_context *kctx, - struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) +static struct kbase_ctx_ext_res_meta * +find_sticky_resource_meta(struct kbase_context *kctx, u64 gpu_addr) { struct kbase_ctx_ext_res_meta *walker; - struct kbase_va_region *reg; lockdep_assert_held(&kctx->reg_lock); - /* Search of the metadata if one isn't provided. */ - if (!meta) { - /* - * Walk the per context external resource metadata list for the - * metadata which matches the region which is being released. - */ - list_for_each_entry(walker, &kctx->ext_res_meta_head, - ext_res_node) { - if (walker->gpu_addr == gpu_addr) { - meta = walker; - break; - } - } - } + /* + * Walk the per context external resource metadata list for the + * metadata which matches the region which is being released. + */ + list_for_each_entry(walker, &kctx->ext_res_meta_head, ext_res_node) + if (walker->gpu_addr == gpu_addr) + return walker; - /* No metadata so just return. */ - if (!meta) - return false; + return NULL; +} + +static void release_sticky_resource_meta(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta) +{ + struct kbase_va_region *reg; /* Drop the physical memory reference and free the metadata. */ reg = kbase_region_tracker_find_region_enclosing_address( @@ -3871,6 +3872,43 @@ bool kbase_sticky_resource_release(struct kbase_context *kctx, kbase_unmap_external_resource(kctx, reg, meta->alloc); list_del(&meta->ext_res_node); kfree(meta); +} + +bool kbase_sticky_resource_release(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) +{ + lockdep_assert_held(&kctx->reg_lock); + + /* Search of the metadata if one isn't provided. */ + if (!meta) + meta = find_sticky_resource_meta(kctx, gpu_addr); + + /* No metadata so just return. */ + if (!meta) + return false; + + if (--meta->ref != 0) + return true; + + release_sticky_resource_meta(kctx, meta); + + return true; +} + +bool kbase_sticky_resource_release_force(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr) +{ + lockdep_assert_held(&kctx->reg_lock); + + /* Search of the metadata if one isn't provided. */ + if (!meta) + meta = find_sticky_resource_meta(kctx, gpu_addr); + + /* No metadata so just return. */ + if (!meta) + return false; + + release_sticky_resource_meta(kctx, meta); return true; } @@ -3901,6 +3939,6 @@ void kbase_sticky_resource_term(struct kbase_context *kctx) walker = list_first_entry(&kctx->ext_res_meta_head, struct kbase_ctx_ext_res_meta, ext_res_node); - kbase_sticky_resource_release(kctx, walker, 0); + kbase_sticky_resource_release_force(kctx, walker, 0); } } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.h b/drivers/gpu/arm/bifrost/mali_kbase_mem.h index bebf55f02727..4fb406db0e2b 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.h @@ -1258,10 +1258,6 @@ int kbasep_find_enclosing_gpu_mapping_start_and_offset( struct kbase_context *kctx, u64 gpu_addr, size_t size, u64 *start, u64 *offset); -enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer); -void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); -void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom); - /** * kbase_alloc_phy_pages_helper - Allocates physical pages. * @alloc: allocation object to add pages to @@ -1593,6 +1589,22 @@ struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire( bool kbase_sticky_resource_release(struct kbase_context *kctx, struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); +/** + * kbase_sticky_resource_release_force - Release a sticky resource. + * @kctx: kbase context. + * @meta: Binding metadata. + * @gpu_addr: GPU address of the external resource. + * + * If meta is NULL then gpu_addr will be used to scan the metadata list and + * find the matching metadata (if any), otherwise the provided meta will be + * used and gpu_addr will be ignored. + * + * Return: True if the release found the metadata and the resource was + * released. + */ +bool kbase_sticky_resource_release_force(struct kbase_context *kctx, + struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr); + /** * kbase_sticky_resource_term - Terminate sticky resource management. * @kctx: kbase context diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c index 8ca31527ca68..71019c68b640 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c @@ -387,14 +387,14 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, unsigned long cookie, cookie_nr; /* Bind to a cookie */ - if (!kctx->cookies) { + if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) { dev_err(dev, "No cookies available for allocation!"); kbase_gpu_vm_unlock(kctx); goto no_cookie; } /* return a cookie */ - cookie_nr = __ffs(kctx->cookies); - kctx->cookies &= ~(1UL << cookie_nr); + cookie_nr = find_first_bit(kctx->cookies, BITS_PER_LONG); + bitmap_clear(kctx->cookies, cookie_nr, 1); BUG_ON(kctx->pending_regions[cookie_nr]); kctx->pending_regions[cookie_nr] = reg; @@ -1740,13 +1740,13 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, #ifdef CONFIG_64BIT if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { /* Bind to a cookie */ - if (!kctx->cookies) { + if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) { dev_err(kctx->kbdev->dev, "No cookies available for allocation!"); goto no_cookie; } /* return a cookie */ - gpu_va = __ffs(kctx->cookies); - kctx->cookies &= ~(1UL << gpu_va); + gpu_va = find_first_bit(kctx->cookies, BITS_PER_LONG); + bitmap_clear(kctx->cookies, gpu_va, 1); BUG_ON(kctx->pending_regions[gpu_va]); kctx->pending_regions[gpu_va] = reg; @@ -1883,11 +1883,11 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, /* mmap needed to setup VA? */ if (*flags & (BASE_MEM_SAME_VA | BASE_MEM_NEED_MMAP)) { /* Bind to a cookie */ - if (!kctx->cookies) + if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) goto no_cookie; /* return a cookie */ - *gpu_va = __ffs(kctx->cookies); - kctx->cookies &= ~(1UL << *gpu_va); + *gpu_va = find_first_bit(kctx->cookies, BITS_PER_LONG); + bitmap_clear(kctx->cookies, *gpu_va, 1); BUG_ON(kctx->pending_regions[*gpu_va]); kctx->pending_regions[*gpu_va] = reg; @@ -2509,7 +2509,7 @@ static int kbasep_reg_mmap(struct kbase_context *kctx, } /* no need for the cookie anymore */ kctx->pending_regions[cookie] = NULL; - kctx->cookies |= (1UL << cookie); + bitmap_set(kctx->cookies, cookie, 1); /* * Overwrite the offset with the region start_pfn, so we effectively diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h index 81e288652b13..d55cc854c415 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h @@ -33,7 +33,7 @@ * @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT */ #define KBASE_MEM_PROFILE_MAX_BUF_SIZE \ - ((size_t) (64 + ((80 + (56 * 64)) * 50) + 56)) + ((size_t) (64 + ((80 + (56 * 64)) * 53) + 56)) #endif /*_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mmu.c b/drivers/gpu/arm/bifrost/mali_kbase_mmu.c index 36210e14d62c..f30d09b62791 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mmu.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mmu.c @@ -279,10 +279,7 @@ static void kbase_gpu_mmu_handle_write_fault(struct kbase_context *kctx, 1, region->flags, region->gpu_alloc->group_id); /* flush L2 and unlock the VA (resumes the MMU) */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367)) - op = AS_COMMAND_FLUSH; - else - op = AS_COMMAND_FLUSH_PT; + op = AS_COMMAND_FLUSH_PT; kbase_gpu_mmu_handle_write_faulting_as(kbdev, faulting_as, fault_pfn, 1, op); @@ -782,10 +779,7 @@ page_fault_retry: mutex_lock(&kbdev->mmu_hw_mutex); /* flush L2 and unlock the VA (resumes the MMU) */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367)) - op = AS_COMMAND_FLUSH; - else - op = AS_COMMAND_FLUSH_PT; + op = AS_COMMAND_FLUSH_PT; /* clear MMU interrupt - this needs to be done after updating * the page tables but before issuing a FLUSH command. The @@ -1449,24 +1443,13 @@ static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx, if (kbase_prepare_to_reset_gpu_locked(kbdev)) kbase_reset_gpu_locked(kbdev); } - -#ifndef CONFIG_MALI_BIFROST_NO_MALI - /* - * As this function could be called in interrupt context the sync - * request can't block. Instead log the request and the next flush - * request will pick it up. - */ - if ((!err) && sync && - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367)) - atomic_set(&kctx->drain_pending, 1); -#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ } /* Perform a flush/invalidate on a particular address space */ static void kbase_mmu_flush_invalidate_as(struct kbase_device *kbdev, struct kbase_as *as, - u64 vpfn, size_t nr, bool sync, bool drain_pending) + u64 vpfn, size_t nr, bool sync) { int err; u32 op; @@ -1501,21 +1484,6 @@ static void kbase_mmu_flush_invalidate_as(struct kbase_device *kbdev, mutex_unlock(&kbdev->mmu_hw_mutex); /* AS transaction end */ -#ifndef CONFIG_MALI_BIFROST_NO_MALI - /* - * The transaction lock must be dropped before here - * as kbase_wait_write_flush could take it if - * the GPU was powered down (static analysis doesn't - * know this can't happen). - */ - drain_pending |= (!err) && sync && - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367); - if (drain_pending) { - /* Wait for GPU to flush write buffer */ - kbase_wait_write_flush(kbdev); - } -#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ - kbase_pm_context_idle(kbdev); } @@ -1525,7 +1493,7 @@ static void kbase_mmu_flush_invalidate_no_ctx(struct kbase_device *kbdev, /* Skip if there is nothing to do */ if (nr) { kbase_mmu_flush_invalidate_as(kbdev, &kbdev->as[as_nr], vpfn, - nr, sync, false); + nr, sync); } } @@ -1534,12 +1502,6 @@ static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, { struct kbase_device *kbdev; bool ctx_is_in_runpool; - bool drain_pending = false; - -#ifndef CONFIG_MALI_BIFROST_NO_MALI - if (atomic_xchg(&kctx->drain_pending, 0)) - drain_pending = true; -#endif /* !CONFIG_MALI_BIFROST_NO_MALI */ /* Early out if there is nothing to do */ if (nr == 0) @@ -1554,7 +1516,7 @@ static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); kbase_mmu_flush_invalidate_as(kbdev, &kbdev->as[kctx->as_nr], - vpfn, nr, sync, drain_pending); + vpfn, nr, sync); kbasep_js_runpool_release_ctx(kbdev, kctx); } @@ -2101,7 +2063,6 @@ void bus_fault_worker(struct work_struct *data) struct kbase_context *kctx; struct kbase_device *kbdev; struct kbase_fault *fault; - bool reset_status = false; faulting_as = container_of(data, struct kbase_as, work_busfault); fault = &faulting_as->bf_data; @@ -2134,14 +2095,6 @@ void bus_fault_worker(struct work_struct *data) } - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { - /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. - * We start the reset before switching to UNMAPPED to ensure that unrelated jobs - * are evicted from the GPU before the switch. - */ - dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); - reset_status = kbase_prepare_to_reset_gpu(kbdev); - } /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */ if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { unsigned long flags; @@ -2166,9 +2119,6 @@ void bus_fault_worker(struct work_struct *data) kbase_pm_context_idle(kbdev); } - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) - kbase_reset_gpu(kbdev); - kbasep_js_runpool_release_ctx(kbdev, kctx); atomic_dec(&kbdev->faults_pending); @@ -2379,8 +2329,6 @@ static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_device *kbdev; struct kbasep_js_device_data *js_devdata; - bool reset_status = false; - as_no = as->number; kbdev = kctx->kbdev; js_devdata = &kbdev->js_data; @@ -2434,14 +2382,7 @@ static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, /* AS transaction begin */ mutex_lock(&kbdev->mmu_hw_mutex); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { - /* Due to H/W issue 8245 we need to reset the GPU after using UNMAPPED mode. - * We start the reset before switching to UNMAPPED to ensure that unrelated jobs - * are evicted from the GPU before the switch. - */ - dev_err(kbdev->dev, "Unhandled page fault. For this GPU version we now soft-reset the GPU as part of page fault recovery."); - reset_status = kbase_prepare_to_reset_gpu(kbdev); - } + /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_mmu_disable(kctx); @@ -2456,165 +2397,6 @@ static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); kbase_mmu_hw_enable_fault(kbdev, as, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); - - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245) && reset_status) - kbase_reset_gpu(kbdev); -} - -void kbasep_as_do_poke(struct work_struct *work) -{ - struct kbase_as *as; - struct kbase_device *kbdev; - unsigned long flags; - - KBASE_DEBUG_ASSERT(work); - as = container_of(work, struct kbase_as, poke_work); - kbdev = container_of(as, struct kbase_device, as[as->number]); - KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); - - /* GPU power will already be active by virtue of the caller holding a JS - * reference on the address space, and will not release it until this worker - * has finished */ - - /* Further to the comment above, we know that while this function is running - * the AS will not be released as before the atom is released this workqueue - * is flushed (in kbase_as_poking_timer_release_atom) - */ - - /* AS transaction begin */ - mutex_lock(&kbdev->mmu_hw_mutex); - /* Force a uTLB invalidate */ - kbase_mmu_hw_do_operation(kbdev, as, 0, 0, - AS_COMMAND_UNLOCK, 0); - mutex_unlock(&kbdev->mmu_hw_mutex); - /* AS transaction end */ - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (as->poke_refcount && - !(as->poke_state & KBASE_AS_POKE_STATE_KILLING_POKE)) { - /* Only queue up the timer if we need it, and we're not trying to kill it */ - hrtimer_start(&as->poke_timer, HR_TIMER_DELAY_MSEC(5), HRTIMER_MODE_REL); - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - -enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer) -{ - struct kbase_as *as; - int queue_work_ret; - - KBASE_DEBUG_ASSERT(NULL != timer); - as = container_of(timer, struct kbase_as, poke_timer); - KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); - - queue_work_ret = queue_work(as->poke_wq, &as->poke_work); - KBASE_DEBUG_ASSERT(queue_work_ret); - return HRTIMER_NORESTART; -} - -/** - * Retain the poking timer on an atom's context (if the atom hasn't already - * done so), and start the timer (if it's not already started). - * - * This must only be called on a context that's scheduled in, and an atom - * that's running on the GPU. - * - * The caller must hold hwaccess_lock - * - * This can be called safely from atomic context - */ -void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - struct kbase_as *as; - - KBASE_DEBUG_ASSERT(kbdev); - KBASE_DEBUG_ASSERT(kctx); - KBASE_DEBUG_ASSERT(katom); - KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (katom->poking) - return; - - katom->poking = 1; - - /* It's safe to work on the as/as_nr without an explicit reference, - * because the caller holds the hwaccess_lock, and the atom itself - * was also running and had already taken a reference */ - as = &kbdev->as[kctx->as_nr]; - - if (++(as->poke_refcount) == 1) { - /* First refcount for poke needed: check if not already in flight */ - if (!as->poke_state) { - /* need to start poking */ - as->poke_state |= KBASE_AS_POKE_STATE_IN_FLIGHT; - queue_work(as->poke_wq, &as->poke_work); - } - } -} - -/** - * If an atom holds a poking timer, release it and wait for it to finish - * - * This must only be called on a context that's scheduled in, and an atom - * that still has a JS reference on the context - * - * This must \b not be called from atomic context, since it can sleep. - */ -void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom) -{ - struct kbase_as *as; - unsigned long flags; - - KBASE_DEBUG_ASSERT(kbdev); - KBASE_DEBUG_ASSERT(kctx); - KBASE_DEBUG_ASSERT(katom); - KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); - - if (!katom->poking) - return; - - as = &kbdev->as[kctx->as_nr]; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - KBASE_DEBUG_ASSERT(as->poke_refcount > 0); - KBASE_DEBUG_ASSERT(as->poke_state & KBASE_AS_POKE_STATE_IN_FLIGHT); - - if (--(as->poke_refcount) == 0) { - as->poke_state |= KBASE_AS_POKE_STATE_KILLING_POKE; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - hrtimer_cancel(&as->poke_timer); - flush_workqueue(as->poke_wq); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - /* Re-check whether it's still needed */ - if (as->poke_refcount) { - int queue_work_ret; - /* Poking still needed: - * - Another retain will not be starting the timer or queueing work, - * because it's still marked as in-flight - * - The hrtimer has finished, and has not started a new timer or - * queued work because it's been marked as killing - * - * So whatever happens now, just queue the work again */ - as->poke_state &= ~((kbase_as_poke_state)KBASE_AS_POKE_STATE_KILLING_POKE); - queue_work_ret = queue_work(as->poke_wq, &as->poke_work); - KBASE_DEBUG_ASSERT(queue_work_ret); - } else { - /* It isn't - so mark it as not in flight, and not killing */ - as->poke_state = 0u; - - /* The poke associated with the atom has now finished. If this is - * also the last atom on the context, then we can guarentee no more - * pokes (and thus no more poking register accesses) will occur on - * the context until new atoms are run */ - } - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - katom->poking = 0; } void kbase_mmu_interrupt_process(struct kbase_device *kbdev, @@ -2644,20 +2426,6 @@ void kbase_mmu_interrupt_process(struct kbase_device *kbdev, KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); } - if (kbase_as_has_bus_fault(as, fault) && - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8245)) { - bool reset_status; - /* - * Reset the GPU, like in bus_fault_worker, in case an - * earlier error hasn't been properly cleared by this - * point. - */ - dev_err(kbdev->dev, "GPU bus error occurred. For this GPU version we now soft-reset as part of bus error recovery\n"); - reset_status = kbase_prepare_to_reset_gpu_locked(kbdev); - if (reset_status) - kbase_reset_gpu_locked(kbdev); - } - return; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c index c679864994a2..4f8a1e93910a 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c @@ -1419,7 +1419,7 @@ static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) gpu_addr)) goto failed_loop; } else - if (!kbase_sticky_resource_release(katom->kctx, NULL, + if (!kbase_sticky_resource_release_force(katom->kctx, NULL, gpu_addr)) failed = true; } @@ -1443,7 +1443,7 @@ failed_loop: u64 const gpu_addr = ext_res->ext_res[i - 1].ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE; - kbase_sticky_resource_release(katom->kctx, NULL, gpu_addr); + kbase_sticky_resource_release_force(katom->kctx, NULL, gpu_addr); --i; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_tracepoints.c b/drivers/gpu/arm/bifrost/mali_kbase_tracepoints.c index 2c551270e211..68c0c1fa42f7 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_tracepoints.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_tracepoints.c @@ -84,6 +84,7 @@ enum tl_msg_id_obj { KBASE_TL_EVENT_ARRAY_END_KCPUQUEUE_ENQUEUE_DEBUGCOPY, KBASE_TL_EVENT_KCPUQUEUE_ENQUEUE_MAP_IMPORT, KBASE_TL_EVENT_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT, + KBASE_TL_EVENT_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE, KBASE_TL_EVENT_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC, KBASE_TL_EVENT_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC, KBASE_TL_EVENT_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC, @@ -104,6 +105,8 @@ enum tl_msg_id_obj { KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_MAP_IMPORT_END, KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START, KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END, + KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START, + KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END, KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_JIT_ALLOC_START, KBASE_TL_EVENT_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, KBASE_TL_EVENT_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, @@ -286,11 +289,11 @@ enum tl_msg_id_aux { "kcpu_queue,ctx") \ TP_DESC(KBASE_TL_EVENT_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL, \ "KCPU Queue enqueues Signal on Fence", \ - "@pL", \ + "@pp", \ "kcpu_queue,fence") \ TP_DESC(KBASE_TL_EVENT_KCPUQUEUE_ENQUEUE_FENCE_WAIT, \ "KCPU Queue enqueues Wait on Fence", \ - "@pL", \ + "@pp", \ "kcpu_queue,fence") \ TP_DESC(KBASE_TL_EVENT_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_WAIT, \ "Begin array of KCPU Queue enqueues Wait on Cross Queue Sync Object", \ @@ -336,6 +339,10 @@ enum tl_msg_id_aux { "KCPU Queue enqueues Unmap Import", \ "@pL", \ "kcpu_queue,map_import_buf_gpu_addr") \ + TP_DESC(KBASE_TL_EVENT_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE, \ + "KCPU Queue enqueues Unmap Import ignoring reference count", \ + "@pL", \ + "kcpu_queue,map_import_buf_gpu_addr") \ TP_DESC(KBASE_TL_EVENT_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC, \ "Begin array of KCPU Queue enqueues JIT Alloc", \ "@p", \ @@ -416,6 +423,14 @@ enum tl_msg_id_aux { "KCPU Queue ends an Unmap Import", \ "@p", \ "kcpu_queue") \ + TP_DESC(KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START, \ + "KCPU Queue starts an Unmap Import ignoring reference count", \ + "@p", \ + "kcpu_queue") \ + TP_DESC(KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END, \ + "KCPU Queue ends an Unmap Import ignoring reference count", \ + "@p", \ + "kcpu_queue") \ TP_DESC(KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_JIT_ALLOC_START, \ "KCPU Queue starts an array of JIT Allocs", \ "@p", \ @@ -1828,7 +1843,7 @@ void __kbase_tlstream_tl_nret_kcpuqueue_ctx( void __kbase_tlstream_tl_event_kcpuqueue_enqueue_fence_signal( struct kbase_tlstream *stream, const void *kcpu_queue, - u64 fence) + const void *fence) { const u32 msg_id = KBASE_TL_EVENT_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL; const size_t msg_size = sizeof(msg_id) + sizeof(u64) @@ -1854,7 +1869,7 @@ void __kbase_tlstream_tl_event_kcpuqueue_enqueue_fence_signal( void __kbase_tlstream_tl_event_kcpuqueue_enqueue_fence_wait( struct kbase_tlstream *stream, const void *kcpu_queue, - u64 fence) + const void *fence) { const u32 msg_id = KBASE_TL_EVENT_KCPUQUEUE_ENQUEUE_FENCE_WAIT; const size_t msg_size = sizeof(msg_id) + sizeof(u64) @@ -2143,6 +2158,32 @@ void __kbase_tlstream_tl_event_kcpuqueue_enqueue_unmap_import( kbase_tlstream_msgbuf_release(stream, acq_flags); } +void __kbase_tlstream_tl_event_kcpuqueue_enqueue_unmap_import_force( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 map_import_buf_gpu_addr) +{ + const u32 msg_id = KBASE_TL_EVENT_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + + sizeof(map_import_buf_gpu_addr) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + pos = kbasep_serialize_bytes(buffer, + pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + void __kbase_tlstream_tl_event_array_begin_kcpuqueue_enqueue_jit_alloc( struct kbase_tlstream *stream, const void *kcpu_queue) @@ -2623,6 +2664,50 @@ void __kbase_tlstream_tl_event_kcpuqueue_execute_unmap_import_end( kbase_tlstream_msgbuf_release(stream, acq_flags); } +void __kbase_tlstream_tl_event_kcpuqueue_execute_unmap_import_force_start( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + +void __kbase_tlstream_tl_event_kcpuqueue_execute_unmap_import_force_end( + struct kbase_tlstream *stream, + const void *kcpu_queue) +{ + const u32 msg_id = KBASE_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(kcpu_queue) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &kcpu_queue, sizeof(kcpu_queue)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + void __kbase_tlstream_tl_event_kcpuqueue_execute_jit_alloc_start( struct kbase_tlstream *stream, const void *kcpu_queue) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_tracepoints.h b/drivers/gpu/arm/bifrost/mali_kbase_tracepoints.h index 27a342ade843..146b67c4cda4 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_tracepoints.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_tracepoints.h @@ -290,11 +290,11 @@ void __kbase_tlstream_tl_nret_kcpuqueue_ctx( void __kbase_tlstream_tl_event_kcpuqueue_enqueue_fence_signal( struct kbase_tlstream *stream, const void *kcpu_queue, - u64 fence); + const void *fence); void __kbase_tlstream_tl_event_kcpuqueue_enqueue_fence_wait( struct kbase_tlstream *stream, const void *kcpu_queue, - u64 fence); + const void *fence); void __kbase_tlstream_tl_event_array_begin_kcpuqueue_enqueue_cqs_wait( struct kbase_tlstream *stream, const void *kcpu_queue); @@ -334,6 +334,10 @@ void __kbase_tlstream_tl_event_kcpuqueue_enqueue_unmap_import( struct kbase_tlstream *stream, const void *kcpu_queue, u64 map_import_buf_gpu_addr); +void __kbase_tlstream_tl_event_kcpuqueue_enqueue_unmap_import_force( + struct kbase_tlstream *stream, + const void *kcpu_queue, + u64 map_import_buf_gpu_addr); void __kbase_tlstream_tl_event_array_begin_kcpuqueue_enqueue_jit_alloc( struct kbase_tlstream *stream, const void *kcpu_queue); @@ -404,6 +408,12 @@ void __kbase_tlstream_tl_event_kcpuqueue_execute_unmap_import_start( void __kbase_tlstream_tl_event_kcpuqueue_execute_unmap_import_end( struct kbase_tlstream *stream, const void *kcpu_queue); +void __kbase_tlstream_tl_event_kcpuqueue_execute_unmap_import_force_start( + struct kbase_tlstream *stream, + const void *kcpu_queue); +void __kbase_tlstream_tl_event_kcpuqueue_execute_unmap_import_force_end( + struct kbase_tlstream *stream, + const void *kcpu_queue); void __kbase_tlstream_tl_event_kcpuqueue_execute_jit_alloc_start( struct kbase_tlstream *stream, const void *kcpu_queue); @@ -1753,6 +1763,27 @@ struct kbase_tlstream; kcpu_queue, map_import_buf_gpu_addr); \ } while (0) +/** + * KBASE_TLSTREAM_TL_EVENT_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE - + * KCPU Queue enqueues Unmap Import ignoring reference count + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + * @map_import_buf_gpu_addr: Map import buffer GPU ptr + */ +#define KBASE_TLSTREAM_TL_EVENT_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ + kbdev, \ + kcpu_queue, \ + map_import_buf_gpu_addr \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_kcpuqueue_enqueue_unmap_import_force( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue, map_import_buf_gpu_addr); \ + } while (0) + /** * KBASE_TLSTREAM_TL_EVENT_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC - * Begin array of KCPU Queue enqueues JIT Alloc @@ -2164,6 +2195,44 @@ struct kbase_tlstream; kcpu_queue); \ } while (0) +/** + * KBASE_TLSTREAM_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START - + * KCPU Queue starts an Unmap Import ignoring reference count + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#define KBASE_TLSTREAM_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_kcpuqueue_execute_unmap_import_force_start( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) + +/** + * KBASE_TLSTREAM_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END - + * KCPU Queue ends an Unmap Import ignoring reference count + * + * @kbdev: Kbase device + * @kcpu_queue: KCPU queue + */ +#define KBASE_TLSTREAM_TL_EVENT_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ + kbdev, \ + kcpu_queue \ + ) \ + do { \ + int enabled = atomic_read(&kbdev->timeline_is_enabled); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_event_kcpuqueue_execute_unmap_import_force_end( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + kcpu_queue); \ + } while (0) + /** * KBASE_TLSTREAM_TL_EVENT_KCPUQUEUE_EXECUTE_JIT_ALLOC_START - * KCPU Queue starts an array of JIT Allocs diff --git a/drivers/gpu/arm/bifrost/mali_kbase_vinstr.c b/drivers/gpu/arm/bifrost/mali_kbase_vinstr.c index 377642d33306..5e3b74d9d79b 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_vinstr.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_vinstr.c @@ -892,7 +892,7 @@ static long kbasep_vinstr_hwcnt_reader_ioctl( cli, (enum base_hwcnt_reader_event)arg); break; default: - WARN_ON(true); + pr_warn("Unknown HWCNT ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); rcode = -EINVAL; break; } diff --git a/drivers/gpu/arm/bifrost/mali_midg_regmap.h b/drivers/gpu/arm/bifrost/mali_midg_regmap.h index f0ec39193c44..6d5f24308dab 100644 --- a/drivers/gpu/arm/bifrost/mali_midg_regmap.h +++ b/drivers/gpu/arm/bifrost/mali_midg_regmap.h @@ -423,14 +423,9 @@ /* End THREAD_* registers */ /* SHADER_CONFIG register */ -#define SC_ALT_COUNTERS (1ul << 3) -#define SC_OVERRIDE_FWD_PIXEL_KILL (1ul << 4) -#define SC_SDC_DISABLE_OQ_DISCARD (1ul << 6) #define SC_LS_ALLOW_ATTR_TYPES (1ul << 16) -#define SC_LS_PAUSEBUFFER_DISABLE (1ul << 16) #define SC_TLS_HASH_ENABLE (1ul << 17) #define SC_LS_ATTR_CHECK_DISABLE (1ul << 18) -#define SC_ENABLE_TEXGRD_FLAGS (1ul << 25) #define SC_VAR_ALGORITHM (1ul << 29) /* End SHADER_CONFIG register */ diff --git a/drivers/gpu/arm/bifrost/platform/meson/Kbuild b/drivers/gpu/arm/bifrost/platform/meson/Kbuild new file mode 100644 index 000000000000..243415b3c53f --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/meson/Kbuild @@ -0,0 +1,24 @@ +# +# (C) COPYRIGHT 2012-2017, 2019 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# SPDX-License-Identifier: GPL-2.0 +# +# + +mali_kbase-y += \ + $(MALI_PLATFORM_DIR)/mali_kbase_config_meson.o \ + $(MALI_PLATFORM_DIR)/mali_kbase_runtime_pm.o diff --git a/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_config_meson.c b/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_config_meson.c new file mode 100644 index 000000000000..8de7a8852029 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_config_meson.c @@ -0,0 +1,41 @@ +/* + * + * (C) COPYRIGHT 2015, 2017, 2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include + +static struct kbase_platform_config dummy_platform_config; + +struct kbase_platform_config *kbase_get_platform_config(void) +{ + return &dummy_platform_config; +} + +#ifndef CONFIG_OF +int kbase_platform_register(void) +{ + return 0; +} + +void kbase_platform_unregister(void) +{ +} +#endif diff --git a/drivers/gpu/arm/bifrost/platform_dummy/mali_ukk_os.h b/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_config_platform.h similarity index 52% rename from drivers/gpu/arm/bifrost/platform_dummy/mali_ukk_os.h rename to drivers/gpu/arm/bifrost/platform/meson/mali_kbase_config_platform.h index ef1ec708edef..c02886fd2187 100644 --- a/drivers/gpu/arm/bifrost/platform_dummy/mali_ukk_os.h +++ b/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_config_platform.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010, 2012-2014 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2017, 2019 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -20,39 +20,27 @@ * */ - +/** + * Power management configuration + * + * Attached value: pointer to @ref kbase_pm_callback_conf + * Default value: See @ref kbase_pm_callback_conf + */ +#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks) /** - * @file mali_ukk_os.h - * Types and definitions that are common for Linux OSs for the kernel side of the - * User-Kernel interface. + * Platform specific configuration functions + * + * Attached value: pointer to @ref kbase_platform_funcs_conf + * Default value: See @ref kbase_platform_funcs_conf */ +#define PLATFORM_FUNCS (NULL) -#ifndef _UKK_OS_H_ /* Linux version */ -#define _UKK_OS_H_ - -#include +extern struct kbase_pm_callback_conf pm_callbacks; /** - * @addtogroup uk_api User-Kernel Interface API - * @{ + * Autosuspend delay + * + * The delay time (in milliseconds) to be used for autosuspend */ - -/** - * @addtogroup uk_api_kernel UKK (Kernel side) - * @{ - */ - -/** - * Internal OS specific data structure associated with each UKK session. Part - * of a ukk_session object. - */ -typedef struct ukkp_session { - int dummy; /**< No internal OS specific data at this time */ -} ukkp_session; - -/** @} end group uk_api_kernel */ - -/** @} end group uk_api */ - -#endif /* _UKK_OS_H__ */ +#define AUTO_SUSPEND_DELAY (100) diff --git a/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_runtime_pm.c b/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_runtime_pm.c new file mode 100644 index 000000000000..5b7378db8687 --- /dev/null +++ b/drivers/gpu/arm/bifrost/platform/meson/mali_kbase_runtime_pm.c @@ -0,0 +1,257 @@ +/* + * + * (C) COPYRIGHT 2015, 2017-2019 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0 + * + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "mali_kbase_config_platform.h" + + +static struct reset_control **resets; +static int nr_resets; + +static int resets_init(struct kbase_device *kbdev) +{ + struct device_node *np; + int i; + int err = 0; + + np = kbdev->dev->of_node; + + nr_resets = of_count_phandle_with_args(np, "resets", "#reset-cells"); + if (nr_resets <= 0) { + dev_err(kbdev->dev, "Failed to get GPU resets from dtb\n"); + return nr_resets; + } + + resets = devm_kcalloc(kbdev->dev, nr_resets, sizeof(*resets), + GFP_KERNEL); + if (!resets) + return -ENOMEM; + + for (i = 0; i < nr_resets; ++i) { + resets[i] = devm_reset_control_get_exclusive_by_index( + kbdev->dev, i); + if (IS_ERR(resets[i])) { + err = PTR_ERR(resets[i]); + nr_resets = i; + break; + } + } + + return err; +} + +static int pm_callback_soft_reset(struct kbase_device *kbdev) +{ + int ret, i; + + if (!resets) { + ret = resets_init(kbdev); + if (ret) + return ret; + } + + for (i = 0; i < nr_resets; ++i) + reset_control_assert(resets[i]); + + udelay(10); + + for (i = 0; i < nr_resets; ++i) + reset_control_deassert(resets[i]); + + udelay(10); + + /* Override Power Management Settings, values from Amlogic KBase */ + kbase_reg_write(kbdev, GPU_CONTROL_REG(PWR_KEY), 0x2968A819); + kbase_reg_write(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), + 0xfff | (0x20 << 16)); + + /* + * RESET_COMPLETED interrupt will be raised, so continue with + * the normal soft reset procedure + */ + return 0; +} + +static void enable_gpu_power_control(struct kbase_device *kbdev) +{ + unsigned int i; + +#if defined(CONFIG_REGULATOR) + for (i = 0; i < kbdev->nr_regulators; i++) { + if (WARN_ON_ONCE(kbdev->regulators[i] == NULL)) + ; + else if (!regulator_is_enabled(kbdev->regulators[i])) + WARN_ON(regulator_enable(kbdev->regulators[i])); + } +#endif + + for (i = 0; i < kbdev->nr_clocks; i++) { + if (WARN_ON_ONCE(kbdev->clocks[i] == NULL)) + ; + else if (!__clk_is_enabled(kbdev->clocks[i])) + WARN_ON(clk_prepare_enable(kbdev->clocks[i])); + } +} + +static void disable_gpu_power_control(struct kbase_device *kbdev) +{ + unsigned int i; + + for (i = 0; i < kbdev->nr_clocks; i++) { + if (WARN_ON_ONCE(kbdev->clocks[i] == NULL)) + ; + else if (__clk_is_enabled(kbdev->clocks[i])) { + clk_disable_unprepare(kbdev->clocks[i]); + WARN_ON(__clk_is_enabled(kbdev->clocks[i])); + } + + } + +#if defined(CONFIG_REGULATOR) + for (i = 0; i < kbdev->nr_regulators; i++) { + if (WARN_ON_ONCE(kbdev->regulators[i] == NULL)) + ; + else if (regulator_is_enabled(kbdev->regulators[i])) + WARN_ON(regulator_disable(kbdev->regulators[i])); + } +#endif +} + +static int pm_callback_power_on(struct kbase_device *kbdev) +{ + int ret = 1; /* Assume GPU has been powered off */ + int error; + + dev_dbg(kbdev->dev, "%s %p\n", __func__, + (void *)kbdev->dev->pm_domain); + + enable_gpu_power_control(kbdev); + + error = pm_runtime_get_sync(kbdev->dev); + if (error == 1) { + /* + * Let core know that the chip has not been + * powered off, so we can save on re-initialization. + */ + ret = 0; + } + + dev_dbg(kbdev->dev, "pm_runtime_get_sync returned %d\n", error); + + return ret; +} + +static void pm_callback_power_off(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "%s\n", __func__); + + pm_runtime_mark_last_busy(kbdev->dev); + pm_runtime_put_autosuspend(kbdev->dev); + +#ifndef KBASE_PM_RUNTIME + disable_gpu_power_control(kbdev); +#endif +} + +#ifdef KBASE_PM_RUNTIME +static int kbase_device_runtime_init(struct kbase_device *kbdev) +{ + int ret = 0; + + dev_dbg(kbdev->dev, "%s\n", __func__); + + pm_runtime_set_autosuspend_delay(kbdev->dev, AUTO_SUSPEND_DELAY); + pm_runtime_use_autosuspend(kbdev->dev); + + pm_runtime_set_active(kbdev->dev); + pm_runtime_enable(kbdev->dev); + + if (!pm_runtime_enabled(kbdev->dev)) { + dev_warn(kbdev->dev, "pm_runtime not enabled"); + ret = -ENOENT; + } + + return ret; +} + +static void kbase_device_runtime_disable(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "%s\n", __func__); + pm_runtime_disable(kbdev->dev); +} +#endif + +static int pm_callback_runtime_on(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "%s\n", __func__); + + enable_gpu_power_control(kbdev); + return 0; +} + +static void pm_callback_runtime_off(struct kbase_device *kbdev) +{ + dev_dbg(kbdev->dev, "%s\n", __func__); + + disable_gpu_power_control(kbdev); +} + +static void pm_callback_resume(struct kbase_device *kbdev) +{ + int ret = pm_callback_runtime_on(kbdev); + + WARN_ON(ret); +} + +static void pm_callback_suspend(struct kbase_device *kbdev) +{ + pm_callback_runtime_off(kbdev); +} + +struct kbase_pm_callback_conf pm_callbacks = { + .power_on_callback = pm_callback_power_on, + .power_off_callback = pm_callback_power_off, + .power_suspend_callback = pm_callback_suspend, + .power_resume_callback = pm_callback_resume, + .soft_reset_callback = pm_callback_soft_reset, +#ifdef KBASE_PM_RUNTIME + .power_runtime_init_callback = kbase_device_runtime_init, + .power_runtime_term_callback = kbase_device_runtime_disable, + .power_runtime_on_callback = pm_callback_runtime_on, + .power_runtime_off_callback = pm_callback_runtime_off, +#else /* KBASE_PM_RUNTIME */ + .power_runtime_init_callback = NULL, + .power_runtime_term_callback = NULL, + .power_runtime_on_callback = NULL, + .power_runtime_off_callback = NULL, +#endif /* KBASE_PM_RUNTIME */ +};