topology.c

  1/*
  2 * arch/arm64/kernel/topology.c
  3 *
  4 * Copyright (C) 2011,2013,2014 Linaro Limited.
  5 *
  6 * Based on the arm32 version written by Vincent Guittot in turn based on
  7 * arch/sh/kernel/topology.c
  8 *
  9 * This file is subject to the terms and conditions of the GNU General Public
 10 * License.  See the file "COPYING" in the main directory of this archive
 11 * for more details.
 12 */
 13
 14#include <linux/acpi.h>
 15#include <linux/arch_topology.h>
 16#include <linux/cacheinfo.h>
 17#include <linux/cpufreq.h>
 18#include <linux/cpu_smt.h>
 19#include <linux/init.h>
 20#include <linux/percpu.h>
 21#include <linux/sched/isolation.h>
 22#include <linux/xarray.h>
 23
 24#include <asm/cpu.h>
 25#include <asm/cputype.h>
 26#include <asm/topology.h>
 27
 28#ifdef CONFIG_ARM64_AMU_EXTN
 29#define read_corecnt()	read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0)
 30#define read_constcnt()	read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0)
 31#else
 32#define read_corecnt()	(0UL)
 33#define read_constcnt()	(0UL)
 34#endif
 35
 36#undef pr_fmt
 37#define pr_fmt(fmt) "AMU: " fmt
 38
 39/*
 40 * Ensure that amu_scale_freq_tick() will return SCHED_CAPACITY_SCALE until
 41 * the CPU capacity and its associated frequency have been correctly
 42 * initialized.
 43 */
 44static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale) =  1UL << (2 * SCHED_CAPACITY_SHIFT);
 45static cpumask_var_t amu_fie_cpus;
 46
 47struct amu_cntr_sample {
 48	u64		arch_const_cycles_prev;
 49	u64		arch_core_cycles_prev;
 50	unsigned long	last_scale_update;
 51};
 52
 53static DEFINE_PER_CPU_SHARED_ALIGNED(struct amu_cntr_sample, cpu_amu_samples);
 54
 55void update_freq_counters_refs(void)
 56{
 57	struct amu_cntr_sample *amu_sample = this_cpu_ptr(&cpu_amu_samples);
 58
 59	amu_sample->arch_core_cycles_prev = read_corecnt();
 60	amu_sample->arch_const_cycles_prev = read_constcnt();
 61}
 62
 63static inline bool freq_counters_valid(int cpu)
 64{
 65	struct amu_cntr_sample *amu_sample = per_cpu_ptr(&cpu_amu_samples, cpu);
 66
 67	if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask))
 68		return false;
 69
 70	if (!cpu_has_amu_feat(cpu)) {
 71		pr_debug("CPU%d: counters are not supported.\n", cpu);
 72		return false;
 73	}
 74
 75	if (unlikely(!amu_sample->arch_const_cycles_prev ||
 76		     !amu_sample->arch_core_cycles_prev)) {
 77		pr_debug("CPU%d: cycle counters are not enabled.\n", cpu);
 78		return false;
 79	}
 80
 81	return true;
 82}
 83
 84void freq_inv_set_max_ratio(int cpu, u64 max_rate)
 85{
 86	u64 ratio, ref_rate = arch_timer_get_rate();
 87
 88	if (unlikely(!max_rate || !ref_rate)) {
 89		WARN_ONCE(1, "CPU%d: invalid maximum or reference frequency.\n",
 90			 cpu);
 91		return;
 92	}
 93
 94	/*
 95	 * Pre-compute the fixed ratio between the frequency of the constant
 96	 * reference counter and the maximum frequency of the CPU.
 97	 *
 98	 *			    ref_rate
 99	 * arch_max_freq_scale =   ---------- * SCHED_CAPACITY_SCALE²
100	 *			    max_rate
101	 *
102	 * We use a factor of 2 * SCHED_CAPACITY_SHIFT -> SCHED_CAPACITY_SCALE²
103	 * in order to ensure a good resolution for arch_max_freq_scale for
104	 * very low reference frequencies (down to the KHz range which should
105	 * be unlikely).
106	 */
107	ratio = ref_rate << (2 * SCHED_CAPACITY_SHIFT);
108	ratio = div64_u64(ratio, max_rate);
109	if (!ratio) {
110		WARN_ONCE(1, "Reference frequency too low.\n");
111		return;
112	}
113
114	WRITE_ONCE(per_cpu(arch_max_freq_scale, cpu), (unsigned long)ratio);
115}
116
117static void amu_scale_freq_tick(void)
118{
119	struct amu_cntr_sample *amu_sample = this_cpu_ptr(&cpu_amu_samples);
120	u64 prev_core_cnt, prev_const_cnt;
121	u64 core_cnt, const_cnt, scale;
122
123	prev_const_cnt = amu_sample->arch_const_cycles_prev;
124	prev_core_cnt = amu_sample->arch_core_cycles_prev;
125
126	update_freq_counters_refs();
127
128	const_cnt = amu_sample->arch_const_cycles_prev;
129	core_cnt = amu_sample->arch_core_cycles_prev;
130
131	/*
132	 * This should not happen unless the AMUs have been reset and the
133	 * counter values have not been restored - unlikely
134	 */
135	if (unlikely(core_cnt <= prev_core_cnt ||
136		     const_cnt <= prev_const_cnt))
137		return;
138
139	/*
140	 *	    /\core    arch_max_freq_scale
141	 * scale =  ------- * --------------------
142	 *	    /\const   SCHED_CAPACITY_SCALE
143	 *
144	 * See validate_cpu_freq_invariance_counters() for details on
145	 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
146	 */
147	scale = core_cnt - prev_core_cnt;
148	scale *= this_cpu_read(arch_max_freq_scale);
149	scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
150			  const_cnt - prev_const_cnt);
151
152	scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
153	this_cpu_write(arch_freq_scale, (unsigned long)scale);
154
155	amu_sample->last_scale_update = jiffies;
156}
157
158static struct scale_freq_data amu_sfd = {
159	.source = SCALE_FREQ_SOURCE_ARCH,
160	.set_freq_scale = amu_scale_freq_tick,
161};
162
163static __always_inline bool amu_fie_cpu_supported(unsigned int cpu)
164{
165	return cpumask_available(amu_fie_cpus) &&
166		cpumask_test_cpu(cpu, amu_fie_cpus);
167}
168
169void arch_cpu_idle_enter(void)
170{
171	unsigned int cpu = smp_processor_id();
172
173	if (!amu_fie_cpu_supported(cpu))
174		return;
175
176	/* Kick in AMU update but only if one has not happened already */
177	if (housekeeping_cpu(cpu, HK_TYPE_TICK) &&
178	    time_is_before_jiffies(per_cpu(cpu_amu_samples.last_scale_update, cpu)))
179		amu_scale_freq_tick();
180}
181
182#define AMU_SAMPLE_EXP_MS	20
183
184int arch_freq_get_on_cpu(int cpu)
185{
186	struct amu_cntr_sample *amu_sample;
187	unsigned int start_cpu = cpu;
188	unsigned long last_update;
189	unsigned int freq = 0;
190	u64 scale;
191
192	if (!amu_fie_cpu_supported(cpu) || !arch_scale_freq_ref(cpu))
193		return -EOPNOTSUPP;
194
195	while (1) {
196
197		amu_sample = per_cpu_ptr(&cpu_amu_samples, cpu);
198
199		last_update = amu_sample->last_scale_update;
200
201		/*
202		 * For those CPUs that are in full dynticks mode, or those that have
203		 * not seen tick for a while, try an alternative source for the counters
204		 * (and thus freq scale), if available, for given policy: this boils
205		 * down to identifying an active cpu within the same freq domain, if any.
206		 */
207		if (!housekeeping_cpu(cpu, HK_TYPE_TICK) ||
208		    time_is_before_jiffies(last_update + msecs_to_jiffies(AMU_SAMPLE_EXP_MS))) {
209			struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
210			int ref_cpu;
211
212			if (!policy)
213				return -EINVAL;
214
215			if (!cpumask_intersects(policy->related_cpus,
216						housekeeping_cpumask(HK_TYPE_TICK))) {
217				cpufreq_cpu_put(policy);
218				return -EOPNOTSUPP;
219			}
220
221			for_each_cpu_wrap(ref_cpu, policy->cpus, cpu + 1) {
222				if (ref_cpu == start_cpu) {
223					/* Prevent verifying same CPU twice */
224					ref_cpu = nr_cpu_ids;
225					break;
226				}
227				if (!idle_cpu(ref_cpu))
228					break;
229			}
230
231			cpufreq_cpu_put(policy);
232
233			if (ref_cpu >= nr_cpu_ids)
234				/* No alternative to pull info from */
235				return -EAGAIN;
236
237			cpu = ref_cpu;
238		} else {
239			break;
240		}
241	}
242	/*
243	 * Reversed computation to the one used to determine
244	 * the arch_freq_scale value
245	 * (see amu_scale_freq_tick for details)
246	 */
247	scale = arch_scale_freq_capacity(cpu);
248	freq = scale * arch_scale_freq_ref(cpu);
249	freq >>= SCHED_CAPACITY_SHIFT;
250	return freq;
251}
252
253static void amu_fie_setup(const struct cpumask *cpus)
254{
255	int cpu;
256
257	/* We are already set since the last insmod of cpufreq driver */
258	if (cpumask_available(amu_fie_cpus) &&
259	    unlikely(cpumask_subset(cpus, amu_fie_cpus)))
260		return;
261
262	for_each_cpu(cpu, cpus)
263		if (!freq_counters_valid(cpu))
264			return;
265
266	if (!cpumask_available(amu_fie_cpus) &&
267	    !zalloc_cpumask_var(&amu_fie_cpus, GFP_KERNEL)) {
268		WARN_ONCE(1, "Failed to allocate FIE cpumask for CPUs[%*pbl]\n",
269			  cpumask_pr_args(cpus));
270		return;
271	}
272
273	cpumask_or(amu_fie_cpus, amu_fie_cpus, cpus);
274
275	topology_set_scale_freq_source(&amu_sfd, amu_fie_cpus);
276
277	pr_debug("CPUs[%*pbl]: counters will be used for FIE.",
278		 cpumask_pr_args(cpus));
279}
280
281static int init_amu_fie_callback(struct notifier_block *nb, unsigned long val,
282				 void *data)
283{
284	struct cpufreq_policy *policy = data;
285
286	if (val == CPUFREQ_CREATE_POLICY)
287		amu_fie_setup(policy->related_cpus);
288
289	/*
290	 * We don't need to handle CPUFREQ_REMOVE_POLICY event as the AMU
291	 * counters don't have any dependency on cpufreq driver once we have
292	 * initialized AMU support and enabled invariance. The AMU counters will
293	 * keep on working just fine in the absence of the cpufreq driver, and
294	 * for the CPUs for which there are no counters available, the last set
295	 * value of arch_freq_scale will remain valid as that is the frequency
296	 * those CPUs are running at.
297	 */
298
299	return 0;
300}
301
302static struct notifier_block init_amu_fie_notifier = {
303	.notifier_call = init_amu_fie_callback,
304};
305
306static int __init init_amu_fie(void)
307{
308	return cpufreq_register_notifier(&init_amu_fie_notifier,
309					CPUFREQ_POLICY_NOTIFIER);
310}
311core_initcall(init_amu_fie);
312
313#ifdef CONFIG_ACPI_CPPC_LIB
314#include <acpi/cppc_acpi.h>
315
316static void cpu_read_corecnt(void *val)
317{
318	/*
319	 * A value of 0 can be returned if the current CPU does not support AMUs
320	 * or if the counter is disabled for this CPU. A return value of 0 at
321	 * counter read is properly handled as an error case by the users of the
322	 * counter.
323	 */
324	*(u64 *)val = read_corecnt();
325}
326
327static void cpu_read_constcnt(void *val)
328{
329	/*
330	 * Return 0 if the current CPU is affected by erratum 2457168. A value
331	 * of 0 is also returned if the current CPU does not support AMUs or if
332	 * the counter is disabled. A return value of 0 at counter read is
333	 * properly handled as an error case by the users of the counter.
334	 */
335	*(u64 *)val = this_cpu_has_cap(ARM64_WORKAROUND_2457168) ?
336		      0UL : read_constcnt();
337}
338
339static inline
340int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val)
341{
342	/*
343	 * Abort call on counterless CPU or when interrupts are
344	 * disabled - can lead to deadlock in smp sync call.
345	 */
346	if (!cpu_has_amu_feat(cpu))
347		return -EOPNOTSUPP;
348
349	if (WARN_ON_ONCE(irqs_disabled()))
350		return -EPERM;
351
352	smp_call_function_single(cpu, func, val, 1);
353
354	return 0;
355}
356
357/*
358 * Refer to drivers/acpi/cppc_acpi.c for the description of the functions
359 * below.
360 */
361bool cpc_ffh_supported(void)
362{
363	int cpu = get_cpu_with_amu_feat();
364
365	/*
366	 * FFH is considered supported if there is at least one present CPU that
367	 * supports AMUs. Using FFH to read core and reference counters for CPUs
368	 * that do not support AMUs, have counters disabled or that are affected
369	 * by errata, will result in a return value of 0.
370	 *
371	 * This is done to allow any enabled and valid counters to be read
372	 * through FFH, knowing that potentially returning 0 as counter value is
373	 * properly handled by the users of these counters.
374	 */
375	if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask))
376		return false;
377
378	return true;
379}
380
381int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val)
382{
383	int ret = -EOPNOTSUPP;
384
385	switch ((u64)reg->address) {
386	case 0x0:
387		ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val);
388		break;
389	case 0x1:
390		ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val);
391		break;
392	}
393
394	if (!ret) {
395		*val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
396				    reg->bit_offset);
397		*val >>= reg->bit_offset;
398	}
399
400	return ret;
401}
402
403int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val)
404{
405	return -EOPNOTSUPP;
406}
407#endif /* CONFIG_ACPI_CPPC_LIB */