shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-05 10:31:46 +09:00
Code Issues Packages Projects Releases Wiki Activity

ANDROID: vendor_hook: Add hooks to support reader optimistic spin in rwsem

Browse Source 
Since upstream commit 617f3ef951 ("locking/rwsem: Remove
reader optimistic spinning"), vendors have seen increased
contention and blocking on rwsems.

There are attempts to actively fix this upstream:
  https://lore.kernel.org/lkml/20240406081126.8030-1-bongkyu7.kim@samsung.com/

But in the meantime, provide vendorhooks so that vendors can
implement their own optimistic spin routine. In doing so,
vendors see improvements in cold launch times on important apps.

Bug: 331742151
Change-Id: I7466413de9ee1293e86f73880931235d7a9142ac
Signed-off-by: xieliujie <xieliujie@oppo.com>
[jstultz: Rewrote commit message]
Signed-off-by: John Stultz <jstultz@google.com>
This commit is contained in:
xieliujie
2024-03-28 18:36:46 +08:00
parent d0c6724b0f
commit 260bfad693
4 changed files with 22 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/android/vendor_hooks.c
View File

@@ -111,6 +111,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_opt_spin_start);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_opt_spin_finish);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_can_spin_on_owner);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sched_show_task);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_direct_rsteal);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_optimistic_rspin);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_enter);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_exit);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mpam_set);

6
include/trace/hooks/rwsem.h
View File

@@ -28,6 +28,12 @@ DECLARE_HOOK(android_vh_alter_rwsem_list_add,
DECLARE_HOOK(android_vh_rwsem_wake_finish,
TP_PROTO(struct rw_semaphore *sem),
TP_ARGS(sem));
DECLARE_HOOK(android_vh_rwsem_direct_rsteal,
TP_PROTO(struct rw_semaphore *sem, bool *steal),
TP_ARGS(sem, steal));
DECLARE_HOOK(android_vh_rwsem_optimistic_rspin,
TP_PROTO(struct rw_semaphore *sem, long *adjustment, bool *rspin),
TP_ARGS(sem, adjustment, rspin));
#endif /* _TRACE_HOOK_RWSEM_H */
/* This part must be outside protection */
#include <trace/define_trace.h>

2
kernel/locking/osq_lock.c
View File

@@ -203,6 +203,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
return false;
}
EXPORT_SYMBOL_GPL(osq_lock);
void osq_unlock(struct optimistic_spin_queue *lock)
{
@@ -230,3 +231,4 @@ void osq_unlock(struct optimistic_spin_queue *lock)
if (next)
WRITE_ONCE(next->locked, 1);
}
EXPORT_SYMBOL_GPL(osq_unlock);

14
kernel/locking/rwsem.c
View File

@@ -1025,6 +1025,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
struct rwsem_waiter waiter;
DEFINE_WAKE_Q(wake_q);
bool already_on_list = false;
bool steal = true;
bool rspin = false;
/*
* To prevent a constant stream of readers from starving a sleeping
@@ -1038,7 +1040,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
/*
* Reader optimistic lock stealing.
*/
if (!(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) {
trace_android_vh_rwsem_direct_rsteal(sem, &steal);
if (steal && !(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) {
rwsem_set_reader_owned(sem);
lockevent_inc(rwsem_rlock_steal);
@@ -1046,7 +1049,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
* Wake up other readers in the wait queue if it is
* the first reader.
*/
if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) {
wake_readers:
if ((rcnt == 1 || rspin) && (count & RWSEM_FLAG_WAITERS)) {
raw_spin_lock_irq(&sem->wait_lock);
if (!list_empty(&sem->wait_list))
rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED,
@@ -1057,6 +1061,12 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
trace_android_vh_record_rwsem_lock_starttime(current, jiffies);
return sem;
}
/*
* Reader optimistic spinning and stealing.
*/
trace_android_vh_rwsem_optimistic_rspin(sem, &adjustment, &rspin);
if (rspin)
goto wake_readers;
queue:
waiter.task = current;