shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-05 10:31:46 +09:00
Code Issues Packages Projects Releases Wiki Activity

BACKPORT: rcu: Fix missing nocb gp wake on rcu_barrier()

Browse Source 
In preparation for RCU lazy changes, wake up the RCU nocb gp thread if
needed after an entrain.  This change prevents the RCU barrier callback
from waiting in the queue for several seconds before the lazy callbacks
in front of it are serviced.

Reported-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
(cherry picked from commit b8f7aca3f0
 https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git rcu/next)

(Backport:
Conflicts:
   kernel/rcu/tree.c
Due to missing 'rcu: Rework rcu_barrier() and callback-migration logic'
Chose not to backport that.)

Bug: 258241771
Bug: 222463781
Test: CQ
Change-Id: Ib55c5886764b74df22531eca35f076ef7acc08dd
Signed-off-by: Joel Fernandes <joelaf@google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/4062165
Reviewed-by: Vineeth Pillai <vineethrp@google.com>
(cherry picked from commit fc6e55ea65dca9cc52bda6081341f3fcc87f6ee7)
[Cherry picked from chromeos-5.15 tree. Minor tweaks to commit message
to match Android style]
Signed-off-by: Qais Yousef <qyousef@google.com>
This commit is contained in:
Frederic Weisbecker
2022-10-16 16:22:53 +00:00
committed by Treehugger Robot
parent fb310d468a
commit 24e6758060
3 changed files with 17 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
kernel/rcu/tree.c
View File

@@ -4003,12 +4003,21 @@ static void rcu_barrier_func(void *cpu_in)
{
uintptr_t cpu = (uintptr_t)cpu_in;
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
bool wake_nocb = false;
bool was_alldone = false;
rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
rdp->barrier_head.func = rcu_barrier_callback;
debug_rcu_head_queue(&rdp->barrier_head);
rcu_nocb_lock(rdp);
/*
* Flush bypass and wakeup rcuog if we add callbacks to an empty regular
* queue. This way we don't wait for bypass timer that can reach seconds
* if it's fully lazy.
*/
was_alldone = rcu_rdp_is_offloaded(rdp) && !rcu_segcblist_pend_cbs(&rdp->cblist);
WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
wake_nocb = was_alldone && rcu_segcblist_pend_cbs(&rdp->cblist);
if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) {
atomic_inc(&rcu_state.barrier_cpu_count);
} else {
@@ -4017,6 +4026,8 @@ static void rcu_barrier_func(void *cpu_in)
rcu_state.barrier_sequence);
}
rcu_nocb_unlock(rdp);
if (wake_nocb)
wake_nocb_gp(rdp, false);
}
/**

1
kernel/rcu/tree.h
View File

@@ -437,6 +437,7 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp);
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);
static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);
static void rcu_init_one_nocb(struct rcu_node *rnp);
static bool wake_nocb_gp(struct rcu_data *rdp, bool force);
static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
unsigned long j);
static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,

5
kernel/rcu/tree_nocb.h
View File

@@ -1449,6 +1449,11 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)
{
}
static bool wake_nocb_gp(struct rcu_data *rdp, bool force)
{
return false;
}
static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
unsigned long j)
{