diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index f742029f9086..7a600198d24b 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -111,6 +111,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_opt_spin_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_opt_spin_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_can_spin_on_owner); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sched_show_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_direct_rsteal); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_optimistic_rspin); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_enter); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_exit); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mpam_set); diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h index a2fb006cd457..4ed664556e8c 100644 --- a/include/trace/hooks/rwsem.h +++ b/include/trace/hooks/rwsem.h @@ -28,6 +28,12 @@ DECLARE_HOOK(android_vh_alter_rwsem_list_add, DECLARE_HOOK(android_vh_rwsem_wake_finish, TP_PROTO(struct rw_semaphore *sem), TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_direct_rsteal, + TP_PROTO(struct rw_semaphore *sem, bool *steal), + TP_ARGS(sem, steal)); +DECLARE_HOOK(android_vh_rwsem_optimistic_rspin, + TP_PROTO(struct rw_semaphore *sem, long *adjustment, bool *rspin), + TP_ARGS(sem, adjustment, rspin)); #endif /* _TRACE_HOOK_RWSEM_H */ /* This part must be outside protection */ #include diff --git a/kernel/locking/osq_lock.c b/kernel/locking/osq_lock.c index d5610ad52b92..b752ec5cc6b0 100644 --- a/kernel/locking/osq_lock.c +++ b/kernel/locking/osq_lock.c @@ -203,6 +203,7 @@ bool osq_lock(struct optimistic_spin_queue *lock) return false; } +EXPORT_SYMBOL_GPL(osq_lock); void osq_unlock(struct optimistic_spin_queue *lock) { @@ -230,3 +231,4 @@ void osq_unlock(struct optimistic_spin_queue *lock) if (next) WRITE_ONCE(next->locked, 1); } +EXPORT_SYMBOL_GPL(osq_unlock); diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index aa85792d352e..a36788e0aa16 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -1025,6 +1025,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat struct rwsem_waiter waiter; DEFINE_WAKE_Q(wake_q); bool already_on_list = false; + bool steal = true; + bool rspin = false; /* * To prevent a constant stream of readers from starving a sleeping @@ -1038,7 +1040,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat /* * Reader optimistic lock stealing. */ - if (!(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) { + trace_android_vh_rwsem_direct_rsteal(sem, &steal); + if (steal && !(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) { rwsem_set_reader_owned(sem); lockevent_inc(rwsem_rlock_steal); @@ -1046,7 +1049,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat * Wake up other readers in the wait queue if it is * the first reader. */ - if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) { +wake_readers: + if ((rcnt == 1 || rspin) && (count & RWSEM_FLAG_WAITERS)) { raw_spin_lock_irq(&sem->wait_lock); if (!list_empty(&sem->wait_list)) rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, @@ -1057,6 +1061,12 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return sem; } + /* + * Reader optimistic spinning and stealing. + */ + trace_android_vh_rwsem_optimistic_rspin(sem, &adjustment, &rspin); + if (rspin) + goto wake_readers; queue: waiter.task = current;