diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index add813b07bb6..33e8b34f7cdc 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -137,6 +137,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_commit_creds); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_exit_creds); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_override_creds); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_revert_creds); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_mutex_lock_starttime); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rtmutex_lock_starttime); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rwsem_lock_starttime); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_pcpu_rwsem_starttime); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_core_rw_nx); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_init_rw_nx); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_permit_before_init); diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h index 36b942b67b7d..8a39613370a3 100644 --- a/include/linux/percpu-rwsem.h +++ b/include/linux/percpu-rwsem.h @@ -9,6 +9,9 @@ #include #include +void _trace_android_vh_record_pcpu_rwsem_starttime( + struct task_struct *tsk, unsigned long settime); + struct percpu_rw_semaphore { struct rcu_sync rss; unsigned int __percpu *read_count; @@ -68,6 +71,7 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *sem) * bleeding the critical section out. */ preempt_enable(); + _trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies); } static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem) @@ -88,14 +92,17 @@ static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem) * bleeding the critical section out. */ - if (ret) + if (ret) { + _trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies); rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); + } return ret; } static inline void percpu_up_read(struct percpu_rw_semaphore *sem) { + _trace_android_vh_record_pcpu_rwsem_starttime(current, 0); rwsem_release(&sem->dep_map, _RET_IP_); preempt_disable(); diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h index 85a7ebc2d4fa..a6b8ac34f281 100644 --- a/include/trace/hooks/dtask.h +++ b/include/trace/hooks/dtask.h @@ -47,6 +47,18 @@ DECLARE_HOOK(android_vh_sched_show_task, TP_PROTO(struct task_struct *task), TP_ARGS(task)); +DECLARE_HOOK(android_vh_record_mutex_lock_starttime, + TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies), + TP_ARGS(tsk, settime_jiffies)); +DECLARE_HOOK(android_vh_record_rtmutex_lock_starttime, + TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies), + TP_ARGS(tsk, settime_jiffies)); +DECLARE_HOOK(android_vh_record_rwsem_lock_starttime, + TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies), + TP_ARGS(tsk, settime_jiffies)); +DECLARE_HOOK(android_vh_record_pcpu_rwsem_starttime, + TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies), + TP_ARGS(tsk, settime_jiffies)); #endif /* _TRACE_HOOK_DTASK_H */ /* This part must be outside protection */ diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 5ef69fbd94b5..52f66448cdac 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -171,8 +171,10 @@ static __always_inline bool __mutex_trylock_fast(struct mutex *lock) unsigned long curr = (unsigned long)current; unsigned long zero = 0UL; - if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) + if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) { + trace_android_vh_record_mutex_lock_starttime(current, jiffies); return true; + } return false; } @@ -539,6 +541,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne */ void __sched mutex_unlock(struct mutex *lock) { + trace_android_vh_record_mutex_lock_starttime(current, 0); #ifndef CONFIG_DEBUG_LOCK_ALLOC if (__mutex_unlock_fast(lock)) return; @@ -613,6 +616,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas if (ww_ctx) ww_mutex_set_context_fastpath(ww, ww_ctx); trace_contention_end(lock, 0); + trace_android_vh_record_mutex_lock_starttime(current, jiffies); preempt_enable(); return 0; } @@ -731,6 +735,7 @@ skip_wait: raw_spin_unlock(&lock->wait_lock); preempt_enable(); + trace_android_vh_record_mutex_lock_starttime(current, jiffies); return 0; err: @@ -1091,8 +1096,10 @@ int __sched mutex_trylock(struct mutex *lock) MUTEX_WARN_ON(lock->magic != lock); locked = __mutex_trylock(lock); - if (locked) + if (locked) { + trace_android_vh_record_mutex_lock_starttime(current, jiffies); mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); + } return locked; } diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index 185bd1c906b0..2bf52ced2526 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -11,6 +11,21 @@ #include #include +#include + +/* + * trace_android_vh_record_pcpu_rwsem_starttime is called in + * include/linux/percpu-rwsem.h by including include/hooks/dtask.h, which + * will result to build-err. So we create + * func:_trace_android_vh_record_pcpu_rwsem_starttime for percpu-rwsem.h to call. + */ +void _trace_android_vh_record_pcpu_rwsem_starttime(struct task_struct *tsk, + unsigned long settime) +{ + trace_android_vh_record_pcpu_rwsem_starttime(tsk, settime); +} +EXPORT_SYMBOL_GPL(_trace_android_vh_record_pcpu_rwsem_starttime); + int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, const char *name, struct lock_class_key *key) { @@ -248,11 +263,13 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem) /* Wait for all active readers to complete. */ rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE); trace_contention_end(sem, 0); + trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies); } EXPORT_SYMBOL_GPL(percpu_down_write); void percpu_up_write(struct percpu_rw_semaphore *sem) { + trace_android_vh_record_pcpu_rwsem_starttime(current, 0); rwsem_release(&sem->dep_map, _RET_IP_); /* diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index cb9fdff76a8a..ea1caa87bb70 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -31,6 +31,8 @@ static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock, ret = __rt_mutex_lock(&lock->rtmutex, state); if (ret) mutex_release(&lock->dep_map, _RET_IP_); + else + trace_android_vh_record_rtmutex_lock_starttime(current, jiffies); return ret; } @@ -123,8 +125,10 @@ int __sched rt_mutex_trylock(struct rt_mutex *lock) return 0; ret = __rt_mutex_trylock(&lock->rtmutex); - if (ret) + if (ret) { + trace_android_vh_record_rtmutex_lock_starttime(current, jiffies); mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); + } return ret; } @@ -137,6 +141,7 @@ EXPORT_SYMBOL_GPL(rt_mutex_trylock); */ void __sched rt_mutex_unlock(struct rt_mutex *lock) { + trace_android_vh_record_rtmutex_lock_starttime(current, 0); mutex_release(&lock->dep_map, _RET_IP_); __rt_mutex_unlock(&lock->rtmutex); } diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index b5766f6ed40e..0e6c93fb1f2d 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -249,6 +249,7 @@ static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) if (!(*cntp & RWSEM_READ_FAILED_MASK)) { rwsem_set_reader_owned(sem); + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return true; } @@ -262,6 +263,7 @@ static inline bool rwsem_write_trylock(struct rw_semaphore *sem) preempt_disable(); if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) { + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); rwsem_set_owner(sem); ret = true; } @@ -1040,6 +1042,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat raw_spin_unlock_irq(&sem->wait_lock); wake_up_q(&wake_q); } + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return sem; } @@ -1109,6 +1112,7 @@ queue: trace_android_vh_rwsem_read_wait_finish(sem); lockevent_inc(rwsem_rlock); trace_contention_end(sem, 0); + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return sem; out_nolock: @@ -1133,6 +1137,7 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) /* do optimistic spinning and steal lock if possible */ if (rwsem_can_spin_on_owner(sem) && rwsem_optimistic_spin(sem)) { /* rwsem_optimistic_spin() implies ACQUIRE on success */ + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return sem; } @@ -1216,6 +1221,7 @@ trylock_again: raw_spin_unlock_irq(&sem->wait_lock); lockevent_inc(rwsem_wlock); trace_contention_end(sem, 0); + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return sem; out_nolock: @@ -1319,6 +1325,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) tmp + RWSEM_READER_BIAS)) { rwsem_set_reader_owned(sem); ret = 1; + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); break; } } @@ -1366,6 +1373,7 @@ static inline void __up_read(struct rw_semaphore *sem) DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); preempt_disable(); + trace_android_vh_record_rwsem_lock_starttime(current, 0); rwsem_clear_reader_owned(sem); tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count); DEBUG_RWSEMS_WARN_ON(tmp < 0, sem); @@ -1393,6 +1401,7 @@ static inline void __up_write(struct rw_semaphore *sem) !rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem); preempt_disable(); + trace_android_vh_record_rwsem_lock_starttime(current, 0); rwsem_clear_owner(sem); tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); preempt_enable();