From ff481da6367e6ab2d54ef27829bd2b54eaee9618 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Thu, 10 May 2018 15:48:06 +0100 Subject: [PATCH 01/13] ANDROID: sched/fair: add arch scaling function for max frequency capping To be able to scale the cpu capacity by this factor introduce a call to the new arch scaling function arch_scale_max_freq_capacity() in update_cpu_capacity() and provide a default implementation which returns SCHED_CAPACITY_SCALE. Another subsystem (e.g. cpufreq) or architectural or platform specific code can overwrite this default implementation, exactly as for frequency and cpu invariance. It has to be enabled by the arch by defining arch_scale_max_freq_capacity to the actual implementation. Change-Id: I770a8b1f4f7340e9e314f71c64a765bf880f4b4d Signed-off-by: Ionela Voinescu Signed-off-by: Dietmar Eggemann ( Fixed conflict with scaling against the PELT-based scale_rt_capacity ) Signed-off-by: Quentin Perret --- kernel/sched/fair.c | 12 ++++++++---- kernel/sched/sched.h | 9 +++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 036be95a87e9..2c2404630199 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7700,10 +7700,9 @@ static inline void init_sd_lb_stats(struct sd_lb_stats *sds) }; } -static unsigned long scale_rt_capacity(struct sched_domain *sd, int cpu) +static unsigned long scale_rt_capacity(int cpu, unsigned long max) { struct rq *rq = cpu_rq(cpu); - unsigned long max = arch_scale_cpu_capacity(cpu); unsigned long used, free; unsigned long irq; @@ -7725,10 +7724,15 @@ static unsigned long scale_rt_capacity(struct sched_domain *sd, int cpu) static void update_cpu_capacity(struct sched_domain *sd, int cpu) { - unsigned long capacity = scale_rt_capacity(sd, cpu); + unsigned long capacity = arch_scale_cpu_capacity(cpu); struct sched_group *sdg = sd->groups; - cpu_rq(cpu)->cpu_capacity_orig = arch_scale_cpu_capacity(cpu); + cpu_rq(cpu)->cpu_capacity_orig = capacity; + + capacity *= arch_scale_max_freq_capacity(sd, cpu); + capacity >>= SCHED_CAPACITY_SHIFT; + + capacity = scale_rt_capacity(cpu, capacity); if (!capacity) capacity = 1; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 802b1f3405f2..b734577b89bf 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1942,6 +1942,15 @@ unsigned long arch_scale_freq_capacity(int cpu) } #endif +#ifndef arch_scale_max_freq_capacity +struct sched_domain; +static __always_inline +unsigned long arch_scale_max_freq_capacity(struct sched_domain *sd, int cpu) +{ + return SCHED_CAPACITY_SCALE; +} +#endif + #ifdef CONFIG_SMP #ifdef CONFIG_PREEMPT From 1bf760e97d559c9c83511165c73c4e10181436b9 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Thu, 10 May 2018 16:52:33 +0100 Subject: [PATCH 02/13] ANDROID: cpufreq: arch_topology: implement max frequency capping Implements the Max Frequency Capping Engine (MFCE) getter function topology_get_max_freq_scale() to provide the scheduler with a maximum frequency scaling correction factor for more accurate cpu capacity handling by being able to deal with max frequency capping. This scaling factor describes the influence of running a cpu with a current maximum frequency (policy) lower than the maximum possible frequency (cpuinfo). The factor is: policy_max_freq(cpu) << SCHED_CAPACITY_SHIFT / cpuinfo_max_freq(cpu) It also implements the MFCE setter function arch_set_max_freq_scale() which is called from cpufreq_set_policy(). Change-Id: I59e52861ee260755ab0518fe1f7183a2e4e3d0fc Signed-off-by: Ionela Voinescu Signed-off-by: Dietmar Eggemann [Trivial cherry-pick issue in cpufreq.c] Signed-off-by: Quentin Perret --- drivers/base/arch_topology.c | 25 ++++++++++++++++++++++++- drivers/cpufreq/cpufreq.c | 8 ++++++++ include/linux/arch_topology.h | 8 ++++++++ include/linux/cpufreq.h | 2 ++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 63c1e76739f1..9fe681e4ff09 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -17,6 +17,8 @@ #include DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE; +DEFINE_PER_CPU(unsigned long, max_cpu_freq); +DEFINE_PER_CPU(unsigned long, max_freq_scale) = SCHED_CAPACITY_SCALE; void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, unsigned long max_freq) @@ -26,8 +28,29 @@ void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, scale = (cur_freq << SCHED_CAPACITY_SHIFT) / max_freq; - for_each_cpu(i, cpus) + for_each_cpu(i, cpus) { per_cpu(freq_scale, i) = scale; + per_cpu(max_cpu_freq, i) = max_freq; + } +} + +void arch_set_max_freq_scale(struct cpumask *cpus, + unsigned long policy_max_freq) +{ + unsigned long scale, max_freq; + int cpu = cpumask_first(cpus); + + if (cpu > nr_cpu_ids) + return; + + max_freq = per_cpu(max_cpu_freq, cpu); + if (!max_freq) + return; + + scale = (policy_max_freq << SCHED_CAPACITY_SHIFT) / max_freq; + + for_each_cpu(cpu, cpus) + per_cpu(max_freq_scale, cpu) = scale; } DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 8dda62367816..a9cd4905522f 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -152,6 +152,12 @@ __weak void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, } EXPORT_SYMBOL_GPL(arch_set_freq_scale); +__weak void arch_set_max_freq_scale(struct cpumask *cpus, + unsigned long policy_max_freq) +{ +} +EXPORT_SYMBOL_GPL(arch_set_max_freq_scale); + /* * This is a generic cpufreq init() routine which can be used by cpufreq * drivers of SMP systems. It will do following: @@ -2407,6 +2413,8 @@ int cpufreq_set_policy(struct cpufreq_policy *policy, policy->max = new_policy->max; trace_cpu_frequency_limits(policy); + arch_set_max_freq_scale(policy->cpus, policy->max); + policy->cached_target_freq = UINT_MAX; pr_debug("new min and max freqs are %u - %u kHz\n", diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index 1cfe05ea1d89..3208e8256763 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -33,4 +33,12 @@ unsigned long topology_get_freq_scale(int cpu) return per_cpu(freq_scale, cpu); } +DECLARE_PER_CPU(unsigned long, max_freq_scale); + +static inline +unsigned long topology_get_max_freq_scale(struct sched_domain *sd, int cpu) +{ + return per_cpu(max_freq_scale, cpu); +} + #endif /* _LINUX_ARCH_TOPOLOGY_H_ */ diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 536a049d7ecc..66bebf55de45 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -984,6 +984,8 @@ extern unsigned int arch_freq_get_on_cpu(int cpu); extern void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, unsigned long max_freq); +extern void arch_set_max_freq_scale(struct cpumask *cpus, + unsigned long policy_max_freq); /* the following are really really optional */ extern struct freq_attr cpufreq_freq_attr_scaling_available_freqs; From f125452fcb9aa79eb125235ba18a73b978b74c45 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Thu, 10 May 2018 16:54:16 +0100 Subject: [PATCH 03/13] ANDROID: arm64: enable max frequency capping Defines arch_scale_max_freq_capacity() to use the topology driver scale function. Change-Id: If7565747ec862e42ac55196240522ef8d22ca67d Signed-off-by: Ionela Voinescu Signed-off-by: Dietmar Eggemann Signed-off-by: Quentin Perret --- arch/arm64/include/asm/topology.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 0524f2438649..8e0a96d71bd6 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -42,6 +42,9 @@ int pcibus_to_node(struct pci_bus *bus); /* Replace task scheduler's default frequency-invariant accounting */ #define arch_scale_freq_capacity topology_get_freq_scale +/* Replace task scheduler's default max-frequency-invariant accounting */ +#define arch_scale_max_freq_capacity topology_get_max_freq_scale + /* Replace task scheduler's default cpu-invariant accounting */ #define arch_scale_cpu_capacity topology_get_cpu_scale From 3dfddd5691d52ddcd6da3428c98e4e85e62f289f Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Thu, 10 May 2018 16:58:04 +0100 Subject: [PATCH 04/13] ANDROID: arm: enable max frequency capping Defines arch_scale_max_freq_capacity() to use the topology driver scale function. Signed-off-by: Ionela Voinescu Signed-off-by: Dietmar Eggemann Signed-off-by: Quentin Perret Change-Id: I79f444399ea3b2948364fde80ccee52a9ece5b9a --- arch/arm/include/asm/topology.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index 2a786f54d8b8..201dc2011c16 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -30,6 +30,9 @@ const struct cpumask *cpu_coregroup_mask(int cpu); /* Replace task scheduler's default frequency-invariant accounting */ #define arch_scale_freq_capacity topology_get_freq_scale +/* Replace task scheduler's default max-frequency-invariant accounting */ +#define arch_scale_max_freq_capacity topology_get_max_freq_scale + /* Replace task scheduler's default cpu-invariant accounting */ #define arch_scale_cpu_capacity topology_get_cpu_scale From bfc73d183074c72d8e90199e03cbcb98c7fc8357 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Sat, 26 Sep 2015 18:19:54 +0100 Subject: [PATCH 05/13] ANDROID: sched: Update max cpu capacity in case of max frequency constraints Wakeup balancing uses cpu capacity awareness and needs to know the system-wide maximum cpu capacity. Patch "sched: Store system-wide maximum cpu capacity in root domain" finds the system-wide maximum cpu capacity during scheduler domain hierarchy setup. This is sufficient as long as maximum frequency invariance is not enabled. If it is enabled, the system-wide maximum cpu capacity can change between scheduler domain hierarchy setups due to frequency capping. The cpu capacity is changed in update_cpu_capacity() which is called in load balance on the lowest scheduler domain hierarchy level. To be able to know if a change in cpu capacity for a certain cpu also has an effect on the system-wide maximum cpu capacity it is normally necessary to iterate over all cpus. This would be way too costly. That's why this patch follows a different approach. The unsigned long max_cpu_capacity value in struct root_domain is replaced with a struct max_cpu_capacity, containing value (the max_cpu_capacity) and cpu (the cpu index of the cpu providing the maximum cpu_capacity). Changes to the system-wide maximum cpu capacity and the cpu index are made if: 1 System-wide maximum cpu capacity < cpu capacity 2 System-wide maximum cpu capacity > cpu capacity and cpu index == cpu There are no changes to the system-wide maximum cpu capacity in all other cases. Atomic read and write access to the pair (max_cpu_capacity.val, max_cpu_capacity.cpu) is enforced by max_cpu_capacity.lock. The access to max_cpu_capacity.val in task_fits_max() is still performed without taking the max_cpu_capacity.lock. The code to set max cpu capacity in build_sched_domains() has been removed because the whole functionality is now provided by update_cpu_capacity() instead. This approach can introduce errors temporarily, e.g. in case the cpu currently providing the max cpu capacity has its cpu capacity lowered due to frequency capping and calls update_cpu_capacity() before any cpu which might provide the max cpu now. Change-Id: I5063befab088fbf49e5d5e484ce0c6ee6165283a Signed-off-by: Ionela Voinescu * Signed-off-by: Dietmar Eggemann (- Fixed cherry-pick issues, and conflict with a0fe2cf086ae "sched/fair: Tune down misfit NOHZ kicks" which makes use of max_cpu_capacity - Squashed "sched/fair: remove printk while schedule is in progress" fix from Caesar Wang ) Signed-off-by: Quentin Perret --- kernel/sched/fair.c | 34 ++++++++++++++++++++++++++++++++-- kernel/sched/sched.h | 10 +++++++++- kernel/sched/topology.c | 15 +++------------ 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2c2404630199..14a70d341349 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6172,7 +6172,7 @@ static int wake_cap(struct task_struct *p, int cpu, int prev_cpu) return 0; min_cap = min(capacity_orig_of(prev_cpu), capacity_orig_of(cpu)); - max_cap = cpu_rq(cpu)->rd->max_cpu_capacity; + max_cap = cpu_rq(cpu)->rd->max_cpu_capacity.val; /* Minimum capacity is close to max, no need to abort wake_affine */ if (max_cap - min_cap < max_cap >> 3) @@ -7722,16 +7722,46 @@ static unsigned long scale_rt_capacity(int cpu, unsigned long max) return scale_irq_capacity(free, irq, max); } +void init_max_cpu_capacity(struct max_cpu_capacity *mcc) { + raw_spin_lock_init(&mcc->lock); + mcc->val = 0; + mcc->cpu = -1; +} + static void update_cpu_capacity(struct sched_domain *sd, int cpu) { unsigned long capacity = arch_scale_cpu_capacity(cpu); struct sched_group *sdg = sd->groups; + struct max_cpu_capacity *mcc; + unsigned long max_capacity; + int max_cap_cpu; + unsigned long flags; cpu_rq(cpu)->cpu_capacity_orig = capacity; capacity *= arch_scale_max_freq_capacity(sd, cpu); capacity >>= SCHED_CAPACITY_SHIFT; + mcc = &cpu_rq(cpu)->rd->max_cpu_capacity; + + raw_spin_lock_irqsave(&mcc->lock, flags); + max_capacity = mcc->val; + max_cap_cpu = mcc->cpu; + + if ((max_capacity > capacity && max_cap_cpu == cpu) || + (max_capacity < capacity)) { + mcc->val = capacity; + mcc->cpu = cpu; +#ifdef CONFIG_SCHED_DEBUG + raw_spin_unlock_irqrestore(&mcc->lock, flags); + printk_deferred(KERN_INFO "CPU%d: update max cpu_capacity %lu\n", + cpu, capacity); + goto skip_unlock; +#endif + } + raw_spin_unlock_irqrestore(&mcc->lock, flags); + +skip_unlock: __attribute__ ((unused)); capacity = scale_rt_capacity(cpu, capacity); if (!capacity) @@ -7836,7 +7866,7 @@ check_cpu_capacity(struct rq *rq, struct sched_domain *sd) static inline int check_misfit_status(struct rq *rq, struct sched_domain *sd) { return rq->misfit_task_load && - (rq->cpu_capacity_orig < rq->rd->max_cpu_capacity || + (rq->cpu_capacity_orig < rq->rd->max_cpu_capacity.val || check_cpu_capacity(rq, sd)); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index b734577b89bf..eb5890920c16 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -710,6 +710,12 @@ struct perf_domain { struct rcu_head rcu; }; +struct max_cpu_capacity { + raw_spinlock_t lock; + unsigned long val; + int cpu; +}; + /* Scheduling group status flags */ #define SG_OVERLOAD 0x1 /* More than one runnable task on a CPU. */ #define SG_OVERUTILIZED 0x2 /* One or more CPUs are over-utilized. */ @@ -768,7 +774,8 @@ struct root_domain { cpumask_var_t rto_mask; struct cpupri cpupri; - unsigned long max_cpu_capacity; + /* Maximum cpu capacity in the system. */ + struct max_cpu_capacity max_cpu_capacity; /* * NULL-terminated list of performance domains intersecting with the @@ -781,6 +788,7 @@ extern struct root_domain def_root_domain; extern struct mutex sched_domains_mutex; extern void init_defrootdomain(void); +extern void init_max_cpu_capacity(struct max_cpu_capacity *mcc); extern int sched_init_domains(const struct cpumask *cpu_map); extern void rq_attach_root(struct rq *rq, struct root_domain *rd); extern void sched_get_rd(struct root_domain *rd); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index f751ce0b783e..618c0dc007e7 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -510,6 +510,9 @@ static int init_rootdomain(struct root_domain *rd) if (cpupri_init(&rd->cpupri) != 0) goto free_cpudl; + + init_max_cpu_capacity(&rd->max_cpu_capacity); + return 0; free_cpudl: @@ -1930,7 +1933,6 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att enum s_alloc alloc_state; struct sched_domain *sd; struct s_data d; - struct rq *rq = NULL; int i, ret = -ENOMEM; struct sched_domain_topology_level *tl_asym; bool has_asym = false; @@ -1993,13 +1995,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att /* Attach the domains */ rcu_read_lock(); for_each_cpu(i, cpu_map) { - rq = cpu_rq(i); sd = *per_cpu_ptr(d.sd, i); - - /* Use READ_ONCE()/WRITE_ONCE() to avoid load/store tearing: */ - if (rq->cpu_capacity_orig > READ_ONCE(d.rd->max_cpu_capacity)) - WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig); - cpu_attach_domain(sd, d.rd, i); } rcu_read_unlock(); @@ -2007,11 +2003,6 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att if (has_asym) static_branch_enable_cpuslocked(&sched_asym_cpucapacity); - if (rq && sched_debug_enabled) { - pr_info("root domain span: %*pbl (max cpu_capacity = %lu)\n", - cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity); - } - ret = 0; error: __free_domain_allocs(&d, alloc_state, cpu_map); From 7846cccba0b0375560d49ce5cd7b36dbbe41532e Mon Sep 17 00:00:00 2001 From: Morten Rasmussen Date: Thu, 2 Jul 2015 17:16:34 +0100 Subject: [PATCH 06/13] ANDROID: sched: Prevent unnecessary active balance of single task in sched group Scenarios with the busiest group having just one task and the local being idle on topologies with sched groups with different numbers of cpus manage to dodge all load-balance bailout conditions resulting the nr_balance_failed counter to be incremented. This eventually causes a pointless active migration of the task. This patch prevents this by not incrementing the counter when the busiest group only has one task. ASYM_PACKING migrations and migrations due to reduced capacity should still take place as these are explicitly captured by need_active_balance(). A better solution would be to not attempt the load-balance in the first place, but that requires significant changes to the order of bailout conditions and statistics gathering. Change-Id: I28f69c72febe0211decbe77b7bc3e48839d3d7b3 cc: Ingo Molnar cc: Peter Zijlstra Signed-off-by: Morten Rasmussen Signed-off-by: Quentin Perret --- kernel/sched/fair.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 14a70d341349..bd58626868be 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7087,6 +7087,7 @@ struct lb_env { int new_dst_cpu; enum cpu_idle_type idle; long imbalance; + unsigned int src_grp_nr_running; /* The set of CPUs under consideration for load-balancing */ struct cpumask *cpus; @@ -8271,6 +8272,8 @@ next_group: if (env->sd->flags & SD_NUMA) env->fbq_type = fbq_classify_group(&sds->busiest_stat); + env->src_grp_nr_running = sds->busiest_stat.sum_nr_running; + if (!env->sd->parent) { struct root_domain *rd = env->dst_rq->rd; @@ -8965,7 +8968,8 @@ more_balance: * excessive cache_hot migrations and active balances. */ if (idle != CPU_NEWLY_IDLE) - sd->nr_balance_failed++; + if (env.src_grp_nr_running > 1) + sd->nr_balance_failed++; if (need_active_balance(&env)) { unsigned long flags; From f2fce2f48cef413b79e286f58e963e75a4729210 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Mon, 26 Jan 2015 19:47:28 +0000 Subject: [PATCH 07/13] ANDROID: sched: Enable idle balance to pull single task towards cpu with higher capacity We do not want to miss out on the ability to pull a single remaining task from a potential source cpu towards an idle destination cpu. Add an extra criteria to need_active_balance() to kick off active load balance if the source cpu is over-utilized and has lower capacity than the destination cpu. Change-Id: Iea3b42b2a0f8d8a4252e42ba67cc33381a4a1075 cc: Ingo Molnar cc: Peter Zijlstra Signed-off-by: Morten Rasmussen Signed-off-by: Dietmar Eggemann (Fixed rebase conflict caused by the introduction of voluntary_active_balance() in "46a745d90585 sched/fair: Fix unnecessary increase of balance interval") Signed-off-by: Quentin Perret --- kernel/sched/fair.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index bd58626868be..719e31a67993 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8750,6 +8750,13 @@ static int need_active_balance(struct lb_env *env) if (voluntary_active_balance(env)) return 1; + if ((capacity_of(env->src_cpu) < capacity_of(env->dst_cpu)) && + env->src_rq->cfs.h_nr_running == 1 && + cpu_overutilized(env->src_cpu) && + !cpu_overutilized(env->dst_cpu)) { + return 1; + } + return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); } From 1a1d22a3d8f3ddf2c30db2aa6b083e67ae1ba732 Mon Sep 17 00:00:00 2001 From: Chris Redpath Date: Fri, 1 Jun 2018 20:34:10 +0100 Subject: [PATCH 08/13] ANDROID: sched/fair: Attempt to improve throughput for asym cap systems In some systems the capacity and group weights line up to defeat all the small imbalance correction conditions in fix_small_imbalance, which can cause bad task placement. Add a new condition if the existing code can't see anything to fix: If we have asymmetric capacity, and there are more tasks than CPUs in the busiest group *and* there are less tasks than CPUs in the local group then we try to pull something. There could be transient small tasks which prevent this from working, but on the whole it is beneficial for those systems with inconvenient capacity/cluster size relationships. Change-Id: Icf81cde215c082a61f816534b7990ccb70aee409 Signed-off-by: Chris Redpath Signed-off-by: Quentin Perret --- kernel/sched/fair.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 719e31a67993..f81d24124e20 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8402,7 +8402,22 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) capa_move /= SCHED_CAPACITY_SCALE; /* Move if we gain throughput */ - if (capa_move > capa_now) + if (capa_move > capa_now) { + env->imbalance = busiest->load_per_task; + return; + } + + /* We can't see throughput improvement with the load-based + * method, but it is possible depending upon group size and + * capacity range that there might still be an underutilized + * cpu available in an asymmetric capacity system. Do one last + * check just in case. + */ + if (env->sd->flags & SD_ASYM_CPUCAPACITY && + busiest->group_type == group_overloaded && + busiest->sum_nr_running > busiest->group_weight && + local->sum_nr_running < local->group_weight && + local->group_capacity < busiest->group_capacity) env->imbalance = busiest->load_per_task; } From 67de53b177068aca219c11cb977e5d27720819c1 Mon Sep 17 00:00:00 2001 From: Chris Redpath Date: Tue, 5 Jun 2018 11:47:57 +0100 Subject: [PATCH 09/13] ANDROID: sched/fair: Don't balance misfits if it would overload local group When load balancing in a system with misfit tasks present, if we always pull a misfit task to the local group this can lead to pulling a running task from a smaller capacity CPUs to a bigger CPU which is busy. In this situation, the pulled task is likely not to get a chance to run before an idle balance on another small CPU pulls it back. This penalises the pulled task as it is stopped for a short amount of time and then likely relocated to a different CPU (since the original CPU just did a NEWLY_IDLE balance and reset the periodic interval). If we only do this unconditionally for NEWLY_IDLE balance, we can be sure that any tasks and load which are present on the local group are related to short-running tasks which we are happy to displace for a longer running task in a system with misfit tasks present. However, other balance types should only pull a task if we think that the local group is underutilized - checking the number of tasks gives us a conservative estimate here since if they were short tasks we would have been doing NEWLY_IDLE balances instead. Change-Id: I710add1ab1139482620b6addc8370ad194791beb Signed-off-by: Chris Redpath Signed-off-by: Quentin Perret --- kernel/sched/fair.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f81d24124e20..6ba3c9cc1108 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8486,8 +8486,18 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s (sds->avg_load - local->avg_load) * local->group_capacity ) / SCHED_CAPACITY_SCALE; - /* Boost imbalance to allow misfit task to be balanced. */ - if (busiest->group_type == group_misfit_task) { + /* Boost imbalance to allow misfit task to be balanced. + * Always do this if we are doing a NEWLY_IDLE balance + * on the assumption that any tasks we have must not be + * long-running (and hence we cannot rely upon load). + * However if we are not idle, we should assume the tasks + * we have are longer running and not override load-based + * calculations above unless we are sure that the local + * group is underutilized. + */ + if (busiest->group_type == group_misfit_task && + (env->idle == CPU_NEWLY_IDLE || + local->sum_nr_running < local->group_weight)) { env->imbalance = max_t(long, env->imbalance, busiest->group_misfit_task_load); } From c3a40105daf569d5a3687a1161e22c5b6e3751dd Mon Sep 17 00:00:00 2001 From: Chris Redpath Date: Tue, 5 Jun 2018 12:21:33 +0100 Subject: [PATCH 10/13] ANDROID: sched/fair: Also do misfit in overloaded groups If we can classify the group as overloaded, that overrides any classification as misfit but we may still have misfit tasks present. Check the rq we're looking at to see if this is the case. Change-Id: Ida8eb66aa625e34de3fe2ee1b0dd8a78926273d8 Signed-off-by: Chris Redpath [Removed stray reference to rq_has_misfit] Signed-off-by: Valentin Schneider Signed-off-by: Quentin Perret --- kernel/sched/fair.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6ba3c9cc1108..d5b1af2a1fa6 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8782,6 +8782,10 @@ static int need_active_balance(struct lb_env *env) return 1; } + if (env->src_grp_type == group_overloaded && env->src_rq->misfit_task_load) + return 1; + + return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); } From 3a77694caae6a083bc3c94c09151367f38799419 Mon Sep 17 00:00:00 2001 From: Quentin Perret Date: Thu, 30 May 2019 10:20:36 +0100 Subject: [PATCH 11/13] FROMLIST: arm64: defconfig: Enable CONFIG_ENERGY_MODEL The recently introduced Energy Model (EM) framework manages power cost tables for the CPUs of the system. Its only user right now is the scheduler, in the context of Energy Aware Scheduling (EAS). However, the EM framework also offers a generic infrastructure that could replace subsystem-specific implementations of the same concepts, as this is the case in the thermal framework. So, in order to prepare the migration of the thermal subsystem to use the EM framework, enable it in the default arm64 defconfig, which is the most commonly used architecture for IPA. This will also compile-in all of the EAS code, although it won't be enabled by default -- EAS requires to use the 'schedutil' CPUFreq governor while arm64 defaults to 'performance'. Change-Id: Iaef37de63d57434f4c12807b233e39f633500b0c Acked-by: Daniel Lezcano Acked-by: Viresh Kumar Signed-off-by: Quentin Perret Message-Id: <20190530092038.12020-2-quentin.perret@arm.com> --- arch/arm64/configs/defconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 0e58ef02880c..ad0e4944a71f 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -71,6 +71,7 @@ CONFIG_COMPAT=y CONFIG_RANDOMIZE_BASE=y CONFIG_HIBERNATION=y CONFIG_WQ_POWER_EFFICIENT_DEFAULT=y +CONFIG_ENERGY_MODEL=y CONFIG_ARM_CPUIDLE=y CONFIG_CPU_FREQ=y CONFIG_CPU_FREQ_STAT=y From e5c4fc9d508f4f1004de57b7fd277ab8ce225211 Mon Sep 17 00:00:00 2001 From: Quentin Perret Date: Thu, 30 May 2019 10:20:37 +0100 Subject: [PATCH 12/13] FROMLIST: thermal: cpu_cooling: Make the power-related code depend on IPA The core CPU cooling infrastructure has power-related functions that have only one client: IPA. Since there can be no user of those functions if IPA is not compiled in, make sure to guard them with checks on CONFIG_THERMAL_GOV_POWER_ALLOCATOR to not waste space unnecessarily. Change-Id: I889c824e1223a3da47d964da01f722fcdf77bd91 Suggested-by: Daniel Lezcano Signed-off-by: Quentin Perret Message-Id: <20190530092038.12020-3-quentin.perret@arm.com> --- drivers/thermal/cpu_cooling.c | 214 +++++++++++++++++----------------- 1 file changed, 104 insertions(+), 110 deletions(-) diff --git a/drivers/thermal/cpu_cooling.c b/drivers/thermal/cpu_cooling.c index 4c5db59a619b..498f59ab64b2 100644 --- a/drivers/thermal/cpu_cooling.c +++ b/drivers/thermal/cpu_cooling.c @@ -46,7 +46,9 @@ */ struct freq_table { u32 frequency; +#ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR u32 power; +#endif }; /** @@ -96,28 +98,6 @@ static DEFINE_IDA(cpufreq_ida); static DEFINE_MUTEX(cooling_list_lock); static LIST_HEAD(cpufreq_cdev_list); -/* Below code defines functions to be used for cpufreq as cooling device */ - -/** - * get_level: Find the level for a particular frequency - * @cpufreq_cdev: cpufreq_cdev for which the property is required - * @freq: Frequency - * - * Return: level corresponding to the frequency. - */ -static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev, - unsigned int freq) -{ - struct freq_table *freq_table = cpufreq_cdev->freq_table; - unsigned long level; - - for (level = 1; level <= cpufreq_cdev->max_level; level++) - if (freq > freq_table[level].frequency) - break; - - return level - 1; -} - /** * cpufreq_thermal_notifier - notifier callback for cpufreq policy change. * @nb: struct notifier_block * with callback info. @@ -171,6 +151,27 @@ static int cpufreq_thermal_notifier(struct notifier_block *nb, return NOTIFY_OK; } +#ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR +/** + * get_level: Find the level for a particular frequency + * @cpufreq_cdev: cpufreq_cdev for which the property is required + * @freq: Frequency + * + * Return: level corresponding to the frequency. + */ +static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev, + unsigned int freq) +{ + struct freq_table *freq_table = cpufreq_cdev->freq_table; + unsigned long level; + + for (level = 1; level <= cpufreq_cdev->max_level; level++) + if (freq > freq_table[level].frequency) + break; + + return level - 1; +} + /** * update_freq_table() - Update the freq table with power numbers * @cpufreq_cdev: the cpufreq cooling device in which to update the table @@ -319,80 +320,6 @@ static u32 get_dynamic_power(struct cpufreq_cooling_device *cpufreq_cdev, return (raw_cpu_power * cpufreq_cdev->last_load) / 100; } -/* cpufreq cooling device callback functions are defined below */ - -/** - * cpufreq_get_max_state - callback function to get the max cooling state. - * @cdev: thermal cooling device pointer. - * @state: fill this variable with the max cooling state. - * - * Callback for the thermal cooling device to return the cpufreq - * max cooling state. - * - * Return: 0 on success, an error code otherwise. - */ -static int cpufreq_get_max_state(struct thermal_cooling_device *cdev, - unsigned long *state) -{ - struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; - - *state = cpufreq_cdev->max_level; - return 0; -} - -/** - * cpufreq_get_cur_state - callback function to get the current cooling state. - * @cdev: thermal cooling device pointer. - * @state: fill this variable with the current cooling state. - * - * Callback for the thermal cooling device to return the cpufreq - * current cooling state. - * - * Return: 0 on success, an error code otherwise. - */ -static int cpufreq_get_cur_state(struct thermal_cooling_device *cdev, - unsigned long *state) -{ - struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; - - *state = cpufreq_cdev->cpufreq_state; - - return 0; -} - -/** - * cpufreq_set_cur_state - callback function to set the current cooling state. - * @cdev: thermal cooling device pointer. - * @state: set this variable to the current cooling state. - * - * Callback for the thermal cooling device to change the cpufreq - * current cooling state. - * - * Return: 0 on success, an error code otherwise. - */ -static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, - unsigned long state) -{ - struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; - unsigned int clip_freq; - - /* Request state should be less than max_level */ - if (WARN_ON(state > cpufreq_cdev->max_level)) - return -EINVAL; - - /* Check if the old cooling action is same as new cooling action */ - if (cpufreq_cdev->cpufreq_state == state) - return 0; - - clip_freq = cpufreq_cdev->freq_table[state].frequency; - cpufreq_cdev->cpufreq_state = state; - cpufreq_cdev->clipped_freq = clip_freq; - - cpufreq_update_policy(cpufreq_cdev->policy->cpu); - - return 0; -} - /** * cpufreq_get_requested_power() - get the current power * @cdev: &thermal_cooling_device pointer @@ -536,22 +463,88 @@ static int cpufreq_power2state(struct thermal_cooling_device *cdev, power); return 0; } +#endif /* CONFIG_THERMAL_GOV_POWER_ALLOCATOR */ + +/* cpufreq cooling device callback functions are defined below */ + +/** + * cpufreq_get_max_state - callback function to get the max cooling state. + * @cdev: thermal cooling device pointer. + * @state: fill this variable with the max cooling state. + * + * Callback for the thermal cooling device to return the cpufreq + * max cooling state. + * + * Return: 0 on success, an error code otherwise. + */ +static int cpufreq_get_max_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; + + *state = cpufreq_cdev->max_level; + return 0; +} + +/** + * cpufreq_get_cur_state - callback function to get the current cooling state. + * @cdev: thermal cooling device pointer. + * @state: fill this variable with the current cooling state. + * + * Callback for the thermal cooling device to return the cpufreq + * current cooling state. + * + * Return: 0 on success, an error code otherwise. + */ +static int cpufreq_get_cur_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; + + *state = cpufreq_cdev->cpufreq_state; + + return 0; +} + +/** + * cpufreq_set_cur_state - callback function to set the current cooling state. + * @cdev: thermal cooling device pointer. + * @state: set this variable to the current cooling state. + * + * Callback for the thermal cooling device to change the cpufreq + * current cooling state. + * + * Return: 0 on success, an error code otherwise. + */ +static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long state) +{ + struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; + unsigned int clip_freq; + + /* Request state should be less than max_level */ + if (WARN_ON(state > cpufreq_cdev->max_level)) + return -EINVAL; + + /* Check if the old cooling action is same as new cooling action */ + if (cpufreq_cdev->cpufreq_state == state) + return 0; + + clip_freq = cpufreq_cdev->freq_table[state].frequency; + cpufreq_cdev->cpufreq_state = state; + cpufreq_cdev->clipped_freq = clip_freq; + + cpufreq_update_policy(cpufreq_cdev->policy->cpu); + + return 0; +} /* Bind cpufreq callbacks to thermal cooling device ops */ static struct thermal_cooling_device_ops cpufreq_cooling_ops = { - .get_max_state = cpufreq_get_max_state, - .get_cur_state = cpufreq_get_cur_state, - .set_cur_state = cpufreq_set_cur_state, -}; - -static struct thermal_cooling_device_ops cpufreq_power_cooling_ops = { .get_max_state = cpufreq_get_max_state, .get_cur_state = cpufreq_get_cur_state, .set_cur_state = cpufreq_set_cur_state, - .get_requested_power = cpufreq_get_requested_power, - .state2power = cpufreq_state2power, - .power2state = cpufreq_power2state, }; /* Notifier for cpufreq policy change */ @@ -659,18 +652,19 @@ __cpufreq_cooling_register(struct device_node *np, pr_debug("%s: freq:%u KHz\n", __func__, freq); } + cooling_ops = &cpufreq_cooling_ops; +#ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR if (capacitance) { ret = update_freq_table(cpufreq_cdev, capacitance); if (ret) { cdev = ERR_PTR(ret); goto remove_ida; } - - cooling_ops = &cpufreq_power_cooling_ops; - } else { - cooling_ops = &cpufreq_cooling_ops; + cooling_ops->get_requested_power = cpufreq_get_requested_power; + cooling_ops->state2power = cpufreq_state2power; + cooling_ops->power2state = cpufreq_power2state; } - +#endif cdev = thermal_of_cooling_device_register(np, dev_name, cpufreq_cdev, cooling_ops); if (IS_ERR(cdev)) From f3e981191920d439f8863591c89b97750bcb9171 Mon Sep 17 00:00:00 2001 From: Quentin Perret Date: Thu, 30 May 2019 10:20:38 +0100 Subject: [PATCH 13/13] FROMLIST: thermal: cpu_cooling: Migrate to using the EM framework The newly introduced Energy Model framework manages power cost tables in a generic way. Moreover, it supports several types of models since the tables can come from DT or firmware (through SCMI) for example. On the other hand, the cpu_cooling subsystem manages its own power cost tables using only DT data. In order to avoid the duplication of data in the kernel, and in order to enable IPA with EMs coming from more than just DT, remove the private tables from cpu_cooling.c and migrate it to using the centralized EM framework. Doing so should have no visible functional impact for existing users of IPA since: - recent extenstions to the the PM_OPP infrastructure enable the registration of EMs in PM_EM using the DT property used by IPA; - the existing upstream cpufreq drivers marked with the 'CPUFREQ_IS_COOLING_DEV' flag all use the aforementioned PM_OPP infrastructure, which means they all support PM_EM. The only two exceptions are qoriq-cpufreq which doesn't in fact use an EM and scmi-cpufreq which doesn't use DT for power costs. For existing users of cpu_cooling, PM_EM tables will contain the exact same power values that IPA used to compute on its own until now. The only new dependency for them is to compile in CONFIG_ENERGY_MODEL. The case where the thermal subsystem is used without an Energy Model (cpufreq_cooling_ops) is handled by looking directly at CPUFreq's frequency table which is already a dependency for cpu_cooling.c anyway. Since the thermal framework expects the cooling states in a particular order, bail out whenever the CPUFreq table is unsorted, since that is fairly uncommon in general, and there are currently no users of cpu_cooling for this use-case. Change-Id: Id29137bab348d4265b866ce87a00c59e46841d13 Acked-by: Viresh Kumar Signed-off-by: Quentin Perret Message-Id: <20190530092038.12020-4-quentin.perret@arm.com> --- drivers/thermal/Kconfig | 1 + drivers/thermal/cpu_cooling.c | 250 ++++++++++++---------------------- 2 files changed, 91 insertions(+), 160 deletions(-) diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 9966364a6deb..340853a3ca48 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -144,6 +144,7 @@ config THERMAL_GOV_USER_SPACE config THERMAL_GOV_POWER_ALLOCATOR bool "Power allocator thermal governor" + depends on ENERGY_MODEL help Enable this to manage platform thermals by dynamically allocating and limiting power to devices. diff --git a/drivers/thermal/cpu_cooling.c b/drivers/thermal/cpu_cooling.c index 498f59ab64b2..83486775e593 100644 --- a/drivers/thermal/cpu_cooling.c +++ b/drivers/thermal/cpu_cooling.c @@ -19,6 +19,7 @@ #include #include #include +#include #include @@ -36,21 +37,6 @@ * ... */ -/** - * struct freq_table - frequency table along with power entries - * @frequency: frequency in KHz - * @power: power in mW - * - * This structure is built when the cooling device registers and helps - * in translating frequency to power and vice versa. - */ -struct freq_table { - u32 frequency; -#ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR - u32 power; -#endif -}; - /** * struct time_in_idle - Idle time stats * @time: previous reading of the absolute time that this cpu was idle @@ -72,7 +58,7 @@ struct time_in_idle { * frequency. * @max_level: maximum cooling level. One less than total number of valid * cpufreq frequencies. - * @freq_table: Freq table in descending order of frequencies + * @em: Reference on the Energy Model of the device * @cdev: thermal_cooling_device pointer to keep track of the * registered cooling device. * @policy: cpufreq policy. @@ -88,7 +74,7 @@ struct cpufreq_cooling_device { unsigned int cpufreq_state; unsigned int clipped_freq; unsigned int max_level; - struct freq_table *freq_table; /* In descending order */ + struct em_perf_domain *em; struct cpufreq_policy *policy; struct list_head node; struct time_in_idle *idle_time; @@ -162,114 +148,40 @@ static int cpufreq_thermal_notifier(struct notifier_block *nb, static unsigned long get_level(struct cpufreq_cooling_device *cpufreq_cdev, unsigned int freq) { - struct freq_table *freq_table = cpufreq_cdev->freq_table; - unsigned long level; + int i; - for (level = 1; level <= cpufreq_cdev->max_level; level++) - if (freq > freq_table[level].frequency) + for (i = cpufreq_cdev->max_level - 1; i >= 0; i--) { + if (freq > cpufreq_cdev->em->table[i].frequency) break; - - return level - 1; -} - -/** - * update_freq_table() - Update the freq table with power numbers - * @cpufreq_cdev: the cpufreq cooling device in which to update the table - * @capacitance: dynamic power coefficient for these cpus - * - * Update the freq table with power numbers. This table will be used in - * cpu_power_to_freq() and cpu_freq_to_power() to convert between power and - * frequency efficiently. Power is stored in mW, frequency in KHz. The - * resulting table is in descending order. - * - * Return: 0 on success, -EINVAL if there are no OPPs for any CPUs, - * or -ENOMEM if we run out of memory. - */ -static int update_freq_table(struct cpufreq_cooling_device *cpufreq_cdev, - u32 capacitance) -{ - struct freq_table *freq_table = cpufreq_cdev->freq_table; - struct dev_pm_opp *opp; - struct device *dev = NULL; - int num_opps = 0, cpu = cpufreq_cdev->policy->cpu, i; - - dev = get_cpu_device(cpu); - if (unlikely(!dev)) { - pr_warn("No cpu device for cpu %d\n", cpu); - return -ENODEV; } - num_opps = dev_pm_opp_get_opp_count(dev); - if (num_opps < 0) - return num_opps; - - /* - * The cpufreq table is also built from the OPP table and so the count - * should match. - */ - if (num_opps != cpufreq_cdev->max_level + 1) { - dev_warn(dev, "Number of OPPs not matching with max_levels\n"); - return -EINVAL; - } - - for (i = 0; i <= cpufreq_cdev->max_level; i++) { - unsigned long freq = freq_table[i].frequency * 1000; - u32 freq_mhz = freq_table[i].frequency / 1000; - u64 power; - u32 voltage_mv; - - /* - * Find ceil frequency as 'freq' may be slightly lower than OPP - * freq due to truncation while converting to kHz. - */ - opp = dev_pm_opp_find_freq_ceil(dev, &freq); - if (IS_ERR(opp)) { - dev_err(dev, "failed to get opp for %lu frequency\n", - freq); - return -EINVAL; - } - - voltage_mv = dev_pm_opp_get_voltage(opp) / 1000; - dev_pm_opp_put(opp); - - /* - * Do the multiplication with MHz and millivolt so as - * to not overflow. - */ - power = (u64)capacitance * freq_mhz * voltage_mv * voltage_mv; - do_div(power, 1000000000); - - /* power is stored in mW */ - freq_table[i].power = power; - } - - return 0; + return cpufreq_cdev->max_level - i - 1; } static u32 cpu_freq_to_power(struct cpufreq_cooling_device *cpufreq_cdev, u32 freq) { int i; - struct freq_table *freq_table = cpufreq_cdev->freq_table; - for (i = 1; i <= cpufreq_cdev->max_level; i++) - if (freq > freq_table[i].frequency) + for (i = cpufreq_cdev->max_level - 1; i >= 0; i--) { + if (freq > cpufreq_cdev->em->table[i].frequency) break; + } - return freq_table[i - 1].power; + return cpufreq_cdev->em->table[i + 1].power; } static u32 cpu_power_to_freq(struct cpufreq_cooling_device *cpufreq_cdev, u32 power) { int i; - struct freq_table *freq_table = cpufreq_cdev->freq_table; - for (i = 1; i <= cpufreq_cdev->max_level; i++) - if (power > freq_table[i].power) + for (i = cpufreq_cdev->max_level - 1; i >= 0; i--) { + if (power > cpufreq_cdev->em->table[i].power) break; + } - return freq_table[i - 1].frequency; + return cpufreq_cdev->em->table[i + 1].frequency; } /** @@ -410,7 +322,7 @@ static int cpufreq_state2power(struct thermal_cooling_device *cdev, struct thermal_zone_device *tz, unsigned long state, u32 *power) { - unsigned int freq, num_cpus; + unsigned int freq, num_cpus, idx; struct cpufreq_cooling_device *cpufreq_cdev = cdev->devdata; /* Request state should be less than max_level */ @@ -419,7 +331,8 @@ static int cpufreq_state2power(struct thermal_cooling_device *cdev, num_cpus = cpumask_weight(cpufreq_cdev->policy->cpus); - freq = cpufreq_cdev->freq_table[state].frequency; + idx = cpufreq_cdev->max_level - state; + freq = cpufreq_cdev->em->table[idx].frequency; *power = cpu_freq_to_power(cpufreq_cdev, freq) * num_cpus; return 0; @@ -463,8 +376,60 @@ static int cpufreq_power2state(struct thermal_cooling_device *cdev, power); return 0; } + +static inline bool em_is_sane(struct cpufreq_cooling_device *cpufreq_cdev, + struct em_perf_domain *em) { + struct cpufreq_policy *policy; + unsigned int nr_levels; + + if (!em) + return false; + + policy = cpufreq_cdev->policy; + if (!cpumask_equal(policy->related_cpus, to_cpumask(em->cpus))) { + pr_err("The span of pd %*pbl is misaligned with cpufreq policy %*pbl\n", + cpumask_pr_args(to_cpumask(em->cpus)), + cpumask_pr_args(policy->related_cpus)); + return false; + } + + nr_levels = cpufreq_cdev->max_level + 1; + if (em->nr_cap_states != nr_levels) { + pr_err("The number of cap states in pd %*pbl (%u) doesn't match the number of cooling levels (%u)\n", + cpumask_pr_args(to_cpumask(em->cpus)), + em->nr_cap_states, nr_levels); + return false; + } + + return true; +} #endif /* CONFIG_THERMAL_GOV_POWER_ALLOCATOR */ +static unsigned int get_state_freq(struct cpufreq_cooling_device *cpufreq_cdev, + unsigned long state) +{ + struct cpufreq_policy *policy; + unsigned long idx; + +#ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR + /* Use the Energy Model table if available */ + if (cpufreq_cdev->em) { + idx = cpufreq_cdev->max_level - state; + return cpufreq_cdev->em->table[idx].frequency; + } +#endif + + /* Otherwise, fallback on the CPUFreq table */ + policy = cpufreq_cdev->policy; + if (policy->freq_table_sorted == CPUFREQ_TABLE_SORTED_ASCENDING) + idx = cpufreq_cdev->max_level - state; + else + idx = state; + + return policy->freq_table[idx].frequency; +} + + /* cpufreq cooling device callback functions are defined below */ /** @@ -530,7 +495,7 @@ static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, if (cpufreq_cdev->cpufreq_state == state) return 0; - clip_freq = cpufreq_cdev->freq_table[state].frequency; + clip_freq = get_state_freq(cpufreq_cdev, state); cpufreq_cdev->cpufreq_state = state; cpufreq_cdev->clipped_freq = clip_freq; @@ -552,26 +517,12 @@ static struct notifier_block thermal_cpufreq_notifier_block = { .notifier_call = cpufreq_thermal_notifier, }; -static unsigned int find_next_max(struct cpufreq_frequency_table *table, - unsigned int prev_max) -{ - struct cpufreq_frequency_table *pos; - unsigned int max = 0; - - cpufreq_for_each_valid_entry(pos, table) { - if (pos->frequency > max && pos->frequency < prev_max) - max = pos->frequency; - } - - return max; -} - /** * __cpufreq_cooling_register - helper function to create cpufreq cooling device * @np: a valid struct device_node to the cooling device device tree node * @policy: cpufreq policy * Normally this should be same as cpufreq policy->related_cpus. - * @capacitance: dynamic power coefficient for these cpus + * @em: Energy Model of the cpufreq policy * * This interface function registers the cpufreq cooling device with the name * "thermal-cpufreq-%x". This api can support multiple instances of cpufreq @@ -583,12 +534,13 @@ static unsigned int find_next_max(struct cpufreq_frequency_table *table, */ static struct thermal_cooling_device * __cpufreq_cooling_register(struct device_node *np, - struct cpufreq_policy *policy, u32 capacitance) + struct cpufreq_policy *policy, + struct em_perf_domain *em) { struct thermal_cooling_device *cdev; struct cpufreq_cooling_device *cpufreq_cdev; char dev_name[THERMAL_NAME_LENGTH]; - unsigned int freq, i, num_cpus; + unsigned int i, num_cpus; int ret; struct thermal_cooling_device_ops *cooling_ops; bool first; @@ -622,55 +574,38 @@ __cpufreq_cooling_register(struct device_node *np, /* max_level is an index, not a counter */ cpufreq_cdev->max_level = i - 1; - cpufreq_cdev->freq_table = kmalloc_array(i, - sizeof(*cpufreq_cdev->freq_table), - GFP_KERNEL); - if (!cpufreq_cdev->freq_table) { - cdev = ERR_PTR(-ENOMEM); - goto free_idle_time; - } - ret = ida_simple_get(&cpufreq_ida, 0, 0, GFP_KERNEL); if (ret < 0) { cdev = ERR_PTR(ret); - goto free_table; + goto free_idle_time; } cpufreq_cdev->id = ret; snprintf(dev_name, sizeof(dev_name), "thermal-cpufreq-%d", cpufreq_cdev->id); - /* Fill freq-table in descending order of frequencies */ - for (i = 0, freq = -1; i <= cpufreq_cdev->max_level; i++) { - freq = find_next_max(policy->freq_table, freq); - cpufreq_cdev->freq_table[i].frequency = freq; - - /* Warn for duplicate entries */ - if (!freq) - pr_warn("%s: table has duplicate entries\n", __func__); - else - pr_debug("%s: freq:%u KHz\n", __func__, freq); - } - cooling_ops = &cpufreq_cooling_ops; #ifdef CONFIG_THERMAL_GOV_POWER_ALLOCATOR - if (capacitance) { - ret = update_freq_table(cpufreq_cdev, capacitance); - if (ret) { - cdev = ERR_PTR(ret); - goto remove_ida; - } + if (em_is_sane(cpufreq_cdev, em)) { + cpufreq_cdev->em = em; cooling_ops->get_requested_power = cpufreq_get_requested_power; cooling_ops->state2power = cpufreq_state2power; cooling_ops->power2state = cpufreq_power2state; - } + } else #endif + if (policy->freq_table_sorted == CPUFREQ_TABLE_UNSORTED) { + pr_err("%s: unsorted frequency tables are not supported\n", + __func__); + cdev = ERR_PTR(-EINVAL); + goto remove_ida; + } + cdev = thermal_of_cooling_device_register(np, dev_name, cpufreq_cdev, cooling_ops); if (IS_ERR(cdev)) goto remove_ida; - cpufreq_cdev->clipped_freq = cpufreq_cdev->freq_table[0].frequency; + cpufreq_cdev->clipped_freq = get_state_freq(cpufreq_cdev, 0); mutex_lock(&cooling_list_lock); /* Register the notifier for first cpufreq cooling device */ @@ -686,8 +621,6 @@ __cpufreq_cooling_register(struct device_node *np, remove_ida: ida_simple_remove(&cpufreq_ida, cpufreq_cdev->id); -free_table: - kfree(cpufreq_cdev->freq_table); free_idle_time: kfree(cpufreq_cdev->idle_time); free_cdev: @@ -709,7 +642,7 @@ free_cdev: struct thermal_cooling_device * cpufreq_cooling_register(struct cpufreq_policy *policy) { - return __cpufreq_cooling_register(NULL, policy, 0); + return __cpufreq_cooling_register(NULL, policy, NULL); } EXPORT_SYMBOL_GPL(cpufreq_cooling_register); @@ -737,7 +670,6 @@ of_cpufreq_cooling_register(struct cpufreq_policy *policy) { struct device_node *np = of_get_cpu_node(policy->cpu, NULL); struct thermal_cooling_device *cdev = NULL; - u32 capacitance = 0; if (!np) { pr_err("cpu_cooling: OF node not available for cpu%d\n", @@ -746,10 +678,9 @@ of_cpufreq_cooling_register(struct cpufreq_policy *policy) } if (of_find_property(np, "#cooling-cells", NULL)) { - of_property_read_u32(np, "dynamic-power-coefficient", - &capacitance); + struct em_perf_domain *em = em_cpu_get(policy->cpu); - cdev = __cpufreq_cooling_register(np, policy, capacitance); + cdev = __cpufreq_cooling_register(np, policy, em); if (IS_ERR(cdev)) { pr_err("cpu_cooling: cpu%d failed to register as cooling device: %ld\n", policy->cpu, PTR_ERR(cdev)); @@ -791,7 +722,6 @@ void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev) thermal_cooling_device_unregister(cdev); ida_simple_remove(&cpufreq_ida, cpufreq_cdev->id); kfree(cpufreq_cdev->idle_time); - kfree(cpufreq_cdev->freq_table); kfree(cpufreq_cdev); } EXPORT_SYMBOL_GPL(cpufreq_cooling_unregister);