From 544ae28cf6b57bcc2050973c1ee4eeaad1f87a1b Mon Sep 17 00:00:00 2001 From: xieliujie Date: Mon, 10 Jul 2023 20:05:46 +0800 Subject: [PATCH] ANDROID: Inherit "user-aware property" across rtmutex. Since upstream commit 715f7f9ece46 ("locking/rtmutex: Squash !RT tasks to DEFAULT_PRIO"), non-rt tasks do not inherit the nice-priority values across rt_mutexes. This removes the minor (and indirect) priority-inheritance that rt-mutexes provided for CFS tasks. Though without priority inheritance, time-bounded priority inversion can occur between CFS tasks of different nice priorities / cgroup limitations. The proxy-execution efforts are a work-in-progress to resolve this upstream, but in the meantime it is left to vendor hooks to provide a near term solution to avoid priority inversion between CFS tasks. In our oem scheduler, if a CFS thread has an "user-aware property", we will always pick it even if it's vruntime is bigger than the smallest one in runqueue. That's why the trace_android_rvh_replace_next_task_fair vendorhook was added previously in commit 53e809978443 ("ANDROID: vendor_hooks: Add hooks for scheduler"). Thus for our oem scheduler, important CFS tasks(like RenderThread) are marked with the "user-aware property" in their struct task_struct. If those tasks are blocked on an rtmutex, we want to allow the "user-aware property" to be inherited to lock owner, so it will be selected to run immediately to release the lock. To support this, we need new hooks to map "user-aware property" into different rtmutex_waiter prio and update the owner's "user-aware property" if needed. Thus these additional vendor hooks are needed. In the future, once an generalized upstream solution for CFS priority inheritance is in place, this will no longer be needed. Bug: 290585456 Change-Id: I6521ed2086b147400a54da6b84a324baf16bc649 Signed-off-by: xieliujie --- drivers/android/vendor_hooks.c | 2 ++ include/trace/hooks/dtask.h | 11 ++++++++++- include/trace/hooks/sched.h | 4 ++++ kernel/locking/rtmutex.c | 6 ++++++ kernel/sched/core.c | 6 ++++-- kernel/sched/vendor_hooks.c | 1 + 6 files changed, 27 insertions(+), 3 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 0a30c8cbe7bd..35baee2a710d 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -88,6 +88,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_send_sig_info); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_wait_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_wait_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_init); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_task_blocks_on_rtmutex); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rtmutex_waiter_prio); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rtmutex_wait_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rtmutex_wait_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_opt_spin_start); diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h index cbf3dfd38d36..1552b71c1792 100644 --- a/include/trace/hooks/dtask.h +++ b/include/trace/hooks/dtask.h @@ -91,7 +91,16 @@ DECLARE_HOOK(android_vh_alter_mutex_list_add, DECLARE_HOOK(android_vh_mutex_unlock_slowpath, TP_PROTO(struct mutex *lock), TP_ARGS(lock)); - +struct rt_mutex_waiter; +struct ww_acquire_ctx; +DECLARE_HOOK(android_vh_task_blocks_on_rtmutex, + TP_PROTO(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter, + struct task_struct *task, struct ww_acquire_ctx *ww_ctx, + unsigned int *chwalk), + TP_ARGS(lock, waiter, task, ww_ctx, chwalk)); +DECLARE_HOOK(android_vh_rtmutex_waiter_prio, + TP_PROTO(struct task_struct *task, int *waiter_prio), + TP_ARGS(task, waiter_prio)); #endif /* _TRACE_HOOK_DTASK_H */ /* This part must be outside protection */ diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h index 811f07f7be61..4cc3f0cded7b 100644 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -52,6 +52,10 @@ DECLARE_RESTRICTED_HOOK(android_rvh_finish_prio_fork, TP_PROTO(struct task_struct *p), TP_ARGS(p), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_rtmutex_force_update, + TP_PROTO(struct task_struct *p, struct task_struct *pi_task, int *update), + TP_ARGS(p, pi_task, update), 1); + DECLARE_RESTRICTED_HOOK(android_rvh_rtmutex_prepare_setprio, TP_PROTO(struct task_struct *p, struct task_struct *pi_task), TP_ARGS(p, pi_task), 1); diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 229ce6bc7d62..351716fe9138 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -327,6 +327,11 @@ static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock, static __always_inline int __waiter_prio(struct task_struct *task) { int prio = task->prio; + int waiter_prio = 0; + + trace_android_vh_rtmutex_waiter_prio(task, &waiter_prio); + if (waiter_prio > 0) + return waiter_prio; if (!rt_prio(prio)) return DEFAULT_PRIO; @@ -1151,6 +1156,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock, if (owner == task && !(build_ww_mutex() && ww_ctx)) return -EDEADLK; + trace_android_vh_task_blocks_on_rtmutex(lock, waiter, task, ww_ctx, &chwalk); raw_spin_lock(&task->pi_lock); waiter->task = task; waiter->lock = lock; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 53faabdb3950..95843540088b 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7043,15 +7043,17 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) const struct sched_class *prev_class; struct rq_flags rf; struct rq *rq; + int update = 0; trace_android_rvh_rtmutex_prepare_setprio(p, pi_task); /* XXX used to be waiter->prio, not waiter->task->prio */ prio = __rt_effective_prio(pi_task, p->normal_prio); + trace_android_rvh_rtmutex_force_update(p, pi_task, &update); /* * If nothing changed; bail early. */ - if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) + if (!update && p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) return; rq = __task_rq_lock(p, &rf); @@ -7071,7 +7073,7 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) /* * For FIFO/RR we only need to set prio, if that matches we're done. */ - if (prio == p->prio && !dl_prio(prio)) + if (!update && prio == p->prio && !dl_prio(prio)) goto out_unlock; /* diff --git a/kernel/sched/vendor_hooks.c b/kernel/sched/vendor_hooks.c index d8d945fc20e3..f528b1f6cbb9 100644 --- a/kernel/sched/vendor_hooks.c +++ b/kernel/sched/vendor_hooks.c @@ -22,6 +22,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_can_migrate_task); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_lowest_rq); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_prepare_prio_fork); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_finish_prio_fork); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_rtmutex_force_update); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_rtmutex_prepare_setprio); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_user_nice); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_setscheduler);