From e270773646d71762ea59d49a957a9b71c1103df9 Mon Sep 17 00:00:00 2001 From: zhujingpeng Date: Tue, 23 Apr 2024 21:50:15 +0800 Subject: [PATCH] ANDROID: vendor_hooks: add hooks in rwsem read trylock When the lock is owned by readers and there is no RWSEM_FLAG_HANDOFF set, we allow some specific new readers to acquire the lock immediately in this hook, event if there are some writer tasks in the wait_list. This features can optimize the priority inversion problem caused by rwsem and improve system responsiveness and performance. Bug: 348699619 Bug: 336506800 Change-Id: I7e8fded73579933d1f61faa9fb6e5f300ffd71bf Signed-off-by: zhujingpeng [jstultz: rebased, resolved collision] Signed-off-by: John Stultz --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/rwsem.h | 3 +++ kernel/locking/rwsem.c | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 9c7a0383c73d..3be1ccdb9cb6 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -420,3 +420,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_tsk_need_resched_lazy); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_read_fault); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_queue_request_and_unlock); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_fuse_request_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_trylock_failed); diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h index 4ed664556e8c..6c1e36468271 100644 --- a/include/trace/hooks/rwsem.h +++ b/include/trace/hooks/rwsem.h @@ -34,6 +34,9 @@ DECLARE_HOOK(android_vh_rwsem_direct_rsteal, DECLARE_HOOK(android_vh_rwsem_optimistic_rspin, TP_PROTO(struct rw_semaphore *sem, long *adjustment, bool *rspin), TP_ARGS(sem, adjustment, rspin)); +DECLARE_HOOK(android_vh_rwsem_read_trylock_failed, + TP_PROTO(struct rw_semaphore *sem, long *cntp, int *ret), + TP_ARGS(sem, cntp, ret)); #endif /* _TRACE_HOOK_RWSEM_H */ /* This part must be outside protection */ #include diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index a36788e0aa16..c847167f536f 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -242,6 +242,8 @@ static inline void rwsem_set_nonspinnable(struct rw_semaphore *sem) static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) { + int ret = 0; + *cntp = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count); if (WARN_ON_ONCE(*cntp < 0)) @@ -253,6 +255,13 @@ static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) return true; } + trace_android_vh_rwsem_read_trylock_failed(sem, cntp, &ret); + if (ret) { + rwsem_set_reader_owned(sem); + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); + return true; + } + return false; } @@ -1354,6 +1363,15 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) break; } } + + if (!ret) { + trace_android_vh_rwsem_read_trylock_failed(sem, NULL, &ret); + if (ret) { + rwsem_set_reader_owned(sem); + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); + } + } + preempt_enable(); return ret; }