From 2870c7853029fda4a34cb2172604d713e6502c04 Mon Sep 17 00:00:00 2001 From: zhujingpeng Date: Mon, 15 Apr 2024 21:22:50 +0800 Subject: [PATCH] ANDROID: GKI: add percpu_rwsem vendor hooks When a writer has set sem->block and is waiting for active readers to complete, we still allow some specific new readers to entry the critical section, which can help prevent priority inversion from impacting system responsiveness and performance. Bug: 334851707 Change-Id: I9e2a7df1efb326763487423d64bcf74d8dec23f8 Signed-off-by: zhujingpeng (cherry picked from commit 458cdb59f7719f81504d392daa95a8c99c30c276) --- drivers/android/vendor_hooks.c | 3 +++ include/trace/hooks/dtask.h | 9 +++++++++ kernel/locking/percpu-rwsem.c | 14 +++++++++++++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 3be1ccdb9cb6..3a317c513c87 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -421,3 +421,6 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_read_fault); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_queue_request_and_unlock); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_fuse_request_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_trylock_failed); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_percpu_rwsem_down_read); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_percpu_rwsem_up_write); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_percpu_rwsem_wait_complete); diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h index ecf3f4b03ab7..08a1ba96413c 100644 --- a/include/trace/hooks/dtask.h +++ b/include/trace/hooks/dtask.h @@ -90,6 +90,15 @@ DECLARE_HOOK(android_vh_record_pcpu_rwsem_time_early, DECLARE_HOOK(android_vh_percpu_rwsem_wq_add, TP_PROTO(struct percpu_rw_semaphore *sem, bool reader), TP_ARGS(sem, reader)); +DECLARE_HOOK(android_vh_percpu_rwsem_down_read, + TP_PROTO(struct percpu_rw_semaphore *sem, bool try, bool *ret), + TP_ARGS(sem, try, ret)); +DECLARE_HOOK(android_vh_percpu_rwsem_up_write, + TP_PROTO(struct percpu_rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_RESTRICTED_HOOK(android_rvh_percpu_rwsem_wait_complete, + TP_PROTO(struct percpu_rw_semaphore *sem, long state, bool *complete), + TP_ARGS(sem, state, complete), 1); struct mutex_waiter; DECLARE_HOOK(android_vh_alter_mutex_list_add, diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index 549da829c954..37f5f1f58ea4 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -196,9 +196,15 @@ static void percpu_rwsem_wait(struct percpu_rw_semaphore *sem, bool reader) bool __sched __percpu_down_read(struct percpu_rw_semaphore *sem, bool try) { + bool ret = false; + if (__percpu_down_read_trylock(sem)) return true; + trace_android_vh_percpu_rwsem_down_read(sem, try, &ret); + if (ret) + return true; + if (try) return false; @@ -253,6 +259,8 @@ static bool readers_active_check(struct percpu_rw_semaphore *sem) void __sched percpu_down_write(struct percpu_rw_semaphore *sem) { + bool complete = false; + might_sleep(); rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); trace_contention_begin(sem, LCB_F_PERCPU | LCB_F_WRITE); @@ -278,7 +286,9 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem) */ /* Wait for all active readers to complete. */ - rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE); + trace_android_rvh_percpu_rwsem_wait_complete(sem, TASK_UNINTERRUPTIBLE, &complete); + if (!complete) + rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE); trace_contention_end(sem, 0); trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies); } @@ -288,6 +298,8 @@ void percpu_up_write(struct percpu_rw_semaphore *sem) { rwsem_release(&sem->dep_map, _RET_IP_); + trace_android_vh_percpu_rwsem_up_write(sem); + /* * Signal the writer is done, no fast path yet. *