diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ac2252bd7c99..fc0d92409249 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6555,6 +6555,44 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) return energy; } +static void select_max_spare_cap_cpus(struct sched_domain *sd, cpumask_t *cpus, + struct perf_domain *pd, struct task_struct *p) +{ + unsigned long spare_cap, max_spare_cap, util, cpu_cap; + int cpu, max_spare_cap_cpu; + + for (; pd; pd = pd->next) { + max_spare_cap_cpu = -1; + max_spare_cap = 0; + + for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) { + if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) + continue; + + /* Skip CPUs that will be overutilized. */ + util = cpu_util_next(cpu, p, cpu); + cpu_cap = capacity_of(cpu); + if (cpu_cap * 1024 < util * capacity_margin) + continue; + + /* + * Find the CPU with the maximum spare capacity in + * the performance domain + */ + spare_cap = cpu_cap - util; + if (spare_cap > max_spare_cap) { + max_spare_cap = spare_cap; + max_spare_cap_cpu = cpu; + } + } + + if (max_spare_cap_cpu >= 0) + cpumask_set_cpu(max_spare_cap_cpu, cpus); + } +} + +static DEFINE_PER_CPU(cpumask_t, energy_cpus); + /* * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the * waking task. find_energy_efficient_cpu() looks for the CPU with maximum @@ -6599,16 +6637,16 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) { unsigned long prev_energy = ULONG_MAX, best_energy = ULONG_MAX; struct root_domain *rd = cpu_rq(smp_processor_id())->rd; - int cpu, best_energy_cpu = prev_cpu; - struct perf_domain *head, *pd; - unsigned long cpu_cap, util; + int weight, cpu, best_energy_cpu = prev_cpu; + unsigned long cur_energy; + struct perf_domain *pd; struct sched_domain *sd; + cpumask_t *candidates; rcu_read_lock(); pd = rcu_dereference(rd->pd); if (!pd || READ_ONCE(rd->overutilized)) goto fail; - head = pd; /* * Energy-aware wake-up happens on the lowest sched_domain starting @@ -6624,45 +6662,29 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) if (!task_util_est(p)) goto unlock; - for (; pd; pd = pd->next) { - unsigned long cur_energy, spare_cap, max_spare_cap = 0; - int max_spare_cap_cpu = -1; + /* Pre-select a set of candidate CPUs. */ + candidates = this_cpu_ptr(&energy_cpus); + cpumask_clear(candidates); + select_max_spare_cap_cpus(sd, candidates, pd, p); - for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) { - if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) - continue; + /* Bail out if there is no candidate, or if the only one is prev_cpu */ + weight = cpumask_weight(candidates); + if (!weight || (weight == 1 && cpumask_first(candidates) == prev_cpu)) + goto unlock; - /* Skip CPUs that will be overutilized. */ - util = cpu_util_next(cpu, p, cpu); - cpu_cap = capacity_of(cpu); - if (cpu_cap * 1024 < util * capacity_margin) - continue; + if (cpumask_test_cpu(prev_cpu, &p->cpus_allowed)) + prev_energy = best_energy = compute_energy(p, prev_cpu, pd); + else + prev_energy = best_energy = ULONG_MAX; - /* Always use prev_cpu as a candidate. */ - if (cpu == prev_cpu) { - prev_energy = compute_energy(p, prev_cpu, head); - best_energy = min(best_energy, prev_energy); - continue; - } - - /* - * Find the CPU with the maximum spare capacity in - * the performance domain - */ - spare_cap = cpu_cap - util; - if (spare_cap > max_spare_cap) { - max_spare_cap = spare_cap; - max_spare_cap_cpu = cpu; - } - } - - /* Evaluate the energy impact of using this CPU. */ - if (max_spare_cap_cpu >= 0) { - cur_energy = compute_energy(p, max_spare_cap_cpu, head); - if (cur_energy < best_energy) { - best_energy = cur_energy; - best_energy_cpu = max_spare_cap_cpu; - } + /* Select the best candidate energy-wise. */ + for_each_cpu(cpu, candidates) { + if (cpu == prev_cpu) + continue; + cur_energy = compute_energy(p, cpu, pd); + if (cur_energy < best_energy) { + best_energy = cur_energy; + best_energy_cpu = cpu; } } unlock: