diff --git a/include/linux/sched.h b/include/linux/sched.h index f9b69056809d..d70914d7d3d4 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -900,6 +900,9 @@ struct task_struct { unsigned sched_reset_on_fork:1; unsigned sched_contributes_to_load:1; unsigned sched_migrated:1; +#ifdef CONFIG_PSI + unsigned sched_psi_wake_requeue:1; +#endif /* Force alignment to the next boundary: */ unsigned :0; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 525718950ec0..45f4965fcb68 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2110,7 +2110,7 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) if (!(flags & ENQUEUE_RESTORE)) { sched_info_enqueue(rq, p); - psi_enqueue(p, (flags & ENQUEUE_WAKEUP) && !(flags & ENQUEUE_MIGRATED)); + psi_enqueue(p, flags & ENQUEUE_WAKEUP); } uclamp_rq_inc(rq, p); diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index b02dfc322951..b49a96fad1d2 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -132,9 +132,11 @@ static inline void psi_enqueue(struct task_struct *p, bool wakeup) if (p->in_memstall) set |= TSK_MEMSTALL_RUNNING; - if (!wakeup) { + if (!wakeup || p->sched_psi_wake_requeue) { if (p->in_memstall) set |= TSK_MEMSTALL; + if (p->sched_psi_wake_requeue) + p->sched_psi_wake_requeue = 0; } else { if (p->in_iowait) clear |= TSK_IOWAIT; @@ -145,6 +147,8 @@ static inline void psi_enqueue(struct task_struct *p, bool wakeup) static inline void psi_dequeue(struct task_struct *p, bool sleep) { + int clear = TSK_RUNNING; + if (static_branch_likely(&psi_disabled)) return; @@ -157,7 +161,10 @@ static inline void psi_dequeue(struct task_struct *p, bool sleep) if (sleep) return; - psi_task_change(p, p->psi_flags, 0); + if (p->in_memstall) + clear |= (TSK_MEMSTALL | TSK_MEMSTALL_RUNNING); + + psi_task_change(p, clear, 0); } static inline void psi_ttwu_dequeue(struct task_struct *p) @@ -169,12 +176,19 @@ static inline void psi_ttwu_dequeue(struct task_struct *p) * deregister its sleep-persistent psi states from the old * queue, and let psi_enqueue() know it has to requeue. */ - if (unlikely(p->psi_flags)) { + if (unlikely(p->in_iowait || p->in_memstall)) { struct rq_flags rf; struct rq *rq; + int clear = 0; + + if (p->in_iowait) + clear |= TSK_IOWAIT; + if (p->in_memstall) + clear |= TSK_MEMSTALL; rq = __task_rq_lock(p, &rf); - psi_task_change(p, p->psi_flags, 0); + psi_task_change(p, clear, 0); + p->sched_psi_wake_requeue = 1; __task_rq_unlock(rq, &rf); } }