diff --git a/include/linux/sched.h b/include/linux/sched.h index 1ea6033e1a09..2291948b702a 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1108,6 +1108,37 @@ unsigned long capacity_curr_of(int cpu); struct sched_group; +struct eas_stats { + /* select_idle_sibling() stats */ + u64 sis_attempts; + u64 sis_idle; + u64 sis_cache_affine; + u64 sis_suff_cap; + u64 sis_idle_cpu; + u64 sis_count; + + /* select_energy_cpu_brute() stats */ + u64 secb_attempts; + u64 secb_sync; + u64 secb_idle_bt; + u64 secb_insuff_cap; + u64 secb_no_nrg_sav; + u64 secb_nrg_sav; + u64 secb_count; + + /* find_best_target() stats */ + u64 fbt_attempts; + u64 fbt_no_cpu; + u64 fbt_no_sd; + u64 fbt_pref_idle; + u64 fbt_count; + + /* cas */ + /* select_task_rq_fair() stats */ + u64 cas_attempts; + u64 cas_count; +}; + struct sched_domain_shared { atomic_t ref; atomic_t nr_busy_cpus; @@ -1176,6 +1207,8 @@ struct sched_domain { unsigned int ttwu_wake_remote; unsigned int ttwu_move_affine; unsigned int ttwu_move_balance; + + struct eas_stats eas_stats; #endif #ifdef CONFIG_SCHED_DEBUG char *name; @@ -1374,6 +1407,35 @@ struct sched_statistics { u64 nr_wakeups_affine_attempts; u64 nr_wakeups_passive; u64 nr_wakeups_idle; + + /* select_idle_sibling() */ + u64 nr_wakeups_sis_attempts; + u64 nr_wakeups_sis_idle; + u64 nr_wakeups_sis_cache_affine; + u64 nr_wakeups_sis_suff_cap; + u64 nr_wakeups_sis_idle_cpu; + u64 nr_wakeups_sis_count; + + /* energy_aware_wake_cpu() */ + u64 nr_wakeups_secb_attempts; + u64 nr_wakeups_secb_sync; + u64 nr_wakeups_secb_idle_bt; + u64 nr_wakeups_secb_insuff_cap; + u64 nr_wakeups_secb_no_nrg_sav; + u64 nr_wakeups_secb_nrg_sav; + u64 nr_wakeups_secb_count; + + /* find_best_target() */ + u64 nr_wakeups_fbt_attempts; + u64 nr_wakeups_fbt_no_cpu; + u64 nr_wakeups_fbt_no_sd; + u64 nr_wakeups_fbt_pref_idle; + u64 nr_wakeups_fbt_count; + + /* cas */ + /* select_task_rq_fair() */ + u64 nr_wakeups_cas_attempts; + u64 nr_wakeups_cas_count; }; #endif diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 48a621c257d1..59e38cd81076 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -981,7 +981,33 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) P_SCHEDSTAT(se.statistics.nr_wakeups_affine_attempts); P_SCHEDSTAT(se.statistics.nr_wakeups_passive); P_SCHEDSTAT(se.statistics.nr_wakeups_idle); - + /* eas */ + /* select_idle_sibling() */ + P_SCHEDSTAT(se.statistics.nr_wakeups_sis_attempts); + P_SCHEDSTAT(se.statistics.nr_wakeups_sis_idle); + P_SCHEDSTAT(se.statistics.nr_wakeups_sis_cache_affine); + P_SCHEDSTAT(se.statistics.nr_wakeups_sis_suff_cap); + P_SCHEDSTAT(se.statistics.nr_wakeups_sis_idle_cpu); + P_SCHEDSTAT(se.statistics.nr_wakeups_sis_count); + /* select_energy_cpu_brute() */ + P_SCHEDSTAT(se.statistics.nr_wakeups_secb_attempts); + P_SCHEDSTAT(se.statistics.nr_wakeups_secb_sync); + P_SCHEDSTAT(se.statistics.nr_wakeups_secb_idle_bt); + P_SCHEDSTAT(se.statistics.nr_wakeups_secb_insuff_cap); + P_SCHEDSTAT(se.statistics.nr_wakeups_secb_no_nrg_sav); + P_SCHEDSTAT(se.statistics.nr_wakeups_secb_nrg_sav); + P_SCHEDSTAT(se.statistics.nr_wakeups_secb_count); + /* find_best_target() */ + P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_attempts); + P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_no_cpu); + P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_no_sd); + P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_pref_idle); + P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_count); + /* cas */ + /* select_task_rq_fair() */ + P_SCHEDSTAT(se.statistics.nr_wakeups_cas_attempts); + P_SCHEDSTAT(se.statistics.nr_wakeups_cas_count); + avg_atom = p->se.sum_exec_runtime; if (nr_switches) avg_atom = div64_ul(avg_atom, nr_switches); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 9e504e30104e..f91a841ab99d 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6382,15 +6382,24 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) int best_idle_cstate = INT_MAX; unsigned long best_idle_capacity = ULONG_MAX; + schedstat_inc(p->se.statistics.nr_wakeups_sis_attempts); + schedstat_inc(this_rq()->eas_stats.sis_attempts); + if (!sysctl_sched_cstate_aware) { - if (idle_cpu(target)) + if (idle_cpu(target)) { + schedstat_inc(p->se.statistics.nr_wakeups_sis_idle); + schedstat_inc(this_rq()->eas_stats.sis_idle); return target; + } /* * If the prevous cpu is cache affine and idle, don't be stupid. */ - if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) + if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) { + schedstat_inc(p->se.statistics.nr_wakeups_sis_cache_affine); + schedstat_inc(this_rq()->eas_stats.sis_cache_affine); return i; + } sd = rcu_dereference(per_cpu(sd_llc, target)); if (!sd) @@ -6430,8 +6439,12 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (new_usage > capacity_orig || !idle_cpu(i)) goto next; - if (i == target && new_usage <= capacity_curr_of(target)) + if (i == target && new_usage <= capacity_curr_of(target)) { + schedstat_inc(p->se.statistics.nr_wakeups_sis_suff_cap); + schedstat_inc(this_rq()->eas_stats.sis_suff_cap); + schedstat_inc(sd->eas_stats.sis_suff_cap); return target; + } if (idle_idx < best_idle_cstate && capacity_orig <= best_idle_capacity) { @@ -6448,6 +6461,9 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) target = cpumask_first_and(sched_group_cpus(sg), tsk_cpus_allowed(p)); + schedstat_inc(p->se.statistics.nr_wakeups_sis_idle_cpu); + schedstat_inc(this_rq()->eas_stats.sis_idle_cpu); + schedstat_inc(sd->eas_stats.sis_idle_cpu); goto done; } next: @@ -6459,6 +6475,9 @@ next: target = best_idle_cpu; done: + schedstat_inc(p->se.statistics.nr_wakeups_sis_count); + schedstat_inc(this_rq()->eas_stats.sis_count); + return target; } @@ -6485,13 +6504,22 @@ static inline int find_best_target(struct task_struct *p, bool boosted, bool pre struct sched_group *sg; int cpu = start_cpu(boosted); - if (cpu < 0) + schedstat_inc(p->se.statistics.nr_wakeups_fbt_attempts); + schedstat_inc(this_rq()->eas_stats.fbt_attempts); + + if (cpu < 0) { + schedstat_inc(p->se.statistics.nr_wakeups_fbt_no_cpu); + schedstat_inc(this_rq()->eas_stats.fbt_no_cpu); return target_cpu; + } sd = rcu_dereference(per_cpu(sd_ea, cpu)); - if (!sd) + if (!sd) { + schedstat_inc(p->se.statistics.nr_wakeups_fbt_no_sd); + schedstat_inc(this_rq()->eas_stats.fbt_no_sd); return target_cpu; + } sg = sd->groups; @@ -6529,8 +6557,11 @@ static inline int find_best_target(struct task_struct *p, bool boosted, bool pre * Unconditionally favoring tasks that prefer idle cpus to * improve latency. */ - if (idle_cpu(i) && prefer_idle) + if (idle_cpu(i) && prefer_idle) { + schedstat_inc(p->se.statistics.nr_wakeups_fbt_pref_idle); + schedstat_inc(this_rq()->eas_stats.fbt_pref_idle); return i; + } cur_capacity = capacity_curr_of(i); @@ -6567,6 +6598,11 @@ static inline int find_best_target(struct task_struct *p, bool boosted, bool pre if (target_cpu < 0) target_cpu = best_idle_cpu >= 0 ? best_idle_cpu : backup_cpu; + if (target_cpu >= 0) { + schedstat_inc(p->se.statistics.nr_wakeups_fbt_count); + schedstat_inc(this_rq()->eas_stats.fbt_count); + } + return target_cpu; } @@ -6616,11 +6652,17 @@ static int select_energy_cpu_brute(struct task_struct *p, int prev_cpu, int sync int target_cpu = prev_cpu, tmp_target; bool boosted, prefer_idle; + schedstat_inc(p->se.statistics.nr_wakeups_secb_attempts); + schedstat_inc(this_rq()->eas_stats.secb_attempts); + if (sysctl_sched_sync_hint_enable && sync) { int cpu = smp_processor_id(); - if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) + if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) { + schedstat_inc(p->se.statistics.nr_wakeups_secb_sync); + schedstat_inc(this_rq()->eas_stats.secb_sync); return cpu; + } } rcu_read_lock(); @@ -6640,8 +6682,11 @@ static int select_energy_cpu_brute(struct task_struct *p, int prev_cpu, int sync goto unlock; if (tmp_target >= 0) { target_cpu = tmp_target; - if ((boosted || prefer_idle) && idle_cpu(target_cpu)) + if ((boosted || prefer_idle) && idle_cpu(target_cpu)) { + schedstat_inc(p->se.statistics.nr_wakeups_secb_idle_bt); + schedstat_inc(this_rq()->eas_stats.secb_idle_bt); goto unlock; + } } if (target_cpu != prev_cpu) { @@ -6653,13 +6698,27 @@ static int select_energy_cpu_brute(struct task_struct *p, int prev_cpu, int sync }; /* Not enough spare capacity on previous cpu */ - if (cpu_overutilized(prev_cpu)) + if (cpu_overutilized(prev_cpu)) { + schedstat_inc(p->se.statistics.nr_wakeups_secb_insuff_cap); + schedstat_inc(this_rq()->eas_stats.secb_insuff_cap); goto unlock; + } - if (energy_diff(&eenv) >= 0) + if (energy_diff(&eenv) >= 0) { + schedstat_inc(p->se.statistics.nr_wakeups_secb_no_nrg_sav); + schedstat_inc(this_rq()->eas_stats.secb_no_nrg_sav); target_cpu = prev_cpu; + goto unlock; + } + + schedstat_inc(p->se.statistics.nr_wakeups_secb_nrg_sav); + schedstat_inc(this_rq()->eas_stats.secb_nrg_sav); + goto unlock; } + schedstat_inc(p->se.statistics.nr_wakeups_secb_count); + schedstat_inc(this_rq()->eas_stats.secb_count); + unlock: rcu_read_unlock(); return target_cpu; @@ -6726,39 +6785,58 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f if (sd_flag & SD_BALANCE_WAKE) /* XXX always ? */ new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); - } else while (sd) { - struct sched_group *group; - int weight; + } else { + int wu = sd_flag & SD_BALANCE_WAKE; + int cas_cpu = -1; - if (!(sd->flags & sd_flag)) { - sd = sd->child; - continue; + if (wu) { + schedstat_inc(p->se.statistics.nr_wakeups_cas_attempts); + schedstat_inc(this_rq()->eas_stats.cas_attempts); } - group = find_idlest_group(sd, p, cpu, sd_flag); - if (!group) { - sd = sd->child; - continue; + + while (sd) { + struct sched_group *group; + int weight; + + if (wu) + schedstat_inc(sd->eas_stats.cas_attempts); + + if (!(sd->flags & sd_flag)) { + sd = sd->child; + continue; + } + + group = find_idlest_group(sd, p, cpu, sd_flag); + if (!group) { + sd = sd->child; + continue; + } + + new_cpu = find_idlest_cpu(group, p, cpu); + if (new_cpu == -1 || new_cpu == cpu) { + /* Now try balancing at a lower domain level of cpu */ + sd = sd->child; + continue; + } + + /* Now try balancing at a lower domain level of new_cpu */ + cpu = cas_cpu = new_cpu; + weight = sd->span_weight; + sd = NULL; + for_each_domain(cpu, tmp) { + if (weight <= tmp->span_weight) + break; + if (tmp->flags & sd_flag) + sd = tmp; + } + /* while loop will break here if sd == NULL */ } - new_cpu = find_idlest_cpu(group, p, cpu); - if (new_cpu == -1 || new_cpu == cpu) { - /* Now try balancing at a lower domain level of cpu */ - sd = sd->child; - continue; + if (wu && (cas_cpu >= 0)) { + schedstat_inc(p->se.statistics.nr_wakeups_cas_count); + schedstat_inc(this_rq()->eas_stats.cas_count); } - - /* Now try balancing at a lower domain level of new_cpu */ - cpu = new_cpu; - weight = sd->span_weight; - sd = NULL; - for_each_domain(cpu, tmp) { - if (weight <= tmp->span_weight) - break; - if (tmp->flags & sd_flag) - sd = tmp; - } - /* while loop will break here if sd == NULL */ } rcu_read_unlock(); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f0e69ba0d4ed..29560c0e8d54 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -768,6 +768,8 @@ struct rq { /* try_to_wake_up() stats */ unsigned int ttwu_count; unsigned int ttwu_local; + + struct eas_stats eas_stats; #endif #ifdef CONFIG_SMP diff --git a/kernel/sched/stats.c b/kernel/sched/stats.c index 87e2c9f0c33e..61f4fdebef42 100644 --- a/kernel/sched/stats.c +++ b/kernel/sched/stats.c @@ -12,6 +12,26 @@ */ #define SCHEDSTAT_VERSION 15 +static inline void show_easstat(struct seq_file *seq, struct eas_stats *stats) +{ + /* eas-specific runqueue stats */ + seq_printf(seq, "eas %llu %llu %llu %llu %llu %llu ", + stats->sis_attempts, stats->sis_idle, stats->sis_cache_affine, + stats->sis_suff_cap, stats->sis_idle_cpu, stats->sis_count); + + seq_printf(seq, "%llu %llu %llu %llu %llu %llu %llu ", + stats->secb_attempts, stats->secb_sync, stats->secb_idle_bt, + stats->secb_insuff_cap, stats->secb_no_nrg_sav, + stats->secb_nrg_sav, stats->secb_count); + + seq_printf(seq, "%llu %llu %llu %llu %llu ", + stats->fbt_attempts, stats->fbt_no_cpu, stats->fbt_no_sd, + stats->fbt_pref_idle, stats->fbt_count); + + seq_printf(seq, "%llu %llu\n", + stats->cas_attempts, stats->cas_count); +} + static int show_schedstat(struct seq_file *seq, void *v) { int cpu; @@ -39,6 +59,8 @@ static int show_schedstat(struct seq_file *seq, void *v) seq_printf(seq, "\n"); + show_easstat(seq, &rq->eas_stats); + #ifdef CONFIG_SMP /* domain-specific stats */ rcu_read_lock(); @@ -66,6 +88,8 @@ static int show_schedstat(struct seq_file *seq, void *v) sd->sbf_count, sd->sbf_balanced, sd->sbf_pushed, sd->ttwu_wake_remote, sd->ttwu_move_affine, sd->ttwu_move_balance); + + show_easstat(seq, &sd->eas_stats); } rcu_read_unlock(); #endif