cppc_cpufreq.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * CPPC (Collaborative Processor Performance Control) driver for
  4 * interfacing with the CPUfreq layer and governors. See
  5 * cppc_acpi.c for CPPC specific methods.
  6 *
  7 * (C) Copyright 2014, 2015 Linaro Ltd.
  8 * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org>
  9 */
 10
 11#define pr_fmt(fmt)	"CPPC Cpufreq:"	fmt
 12
 13#include <linux/arch_topology.h>
 14#include <linux/kernel.h>
 15#include <linux/module.h>
 16#include <linux/delay.h>
 17#include <linux/cpu.h>
 18#include <linux/cpufreq.h>
 19#include <linux/irq_work.h>
 20#include <linux/kthread.h>
 21#include <linux/time.h>
 22#include <linux/vmalloc.h>
 23#include <uapi/linux/sched/types.h>
 24
 25#include <linux/unaligned.h>
 26
 27#include <acpi/cppc_acpi.h>
 28
 29static struct cpufreq_driver cppc_cpufreq_driver;
 30
 31#ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE
 32static enum {
 33	FIE_UNSET = -1,
 34	FIE_ENABLED,
 35	FIE_DISABLED
 36} fie_disabled = FIE_UNSET;
 37
 38module_param(fie_disabled, int, 0444);
 39MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)");
 40
 41/* Frequency invariance support */
 42struct cppc_freq_invariance {
 43	int cpu;
 44	struct irq_work irq_work;
 45	struct kthread_work work;
 46	struct cppc_perf_fb_ctrs prev_perf_fb_ctrs;
 47	struct cppc_cpudata *cpu_data;
 48};
 49
 50static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv);
 51static struct kthread_worker *kworker_fie;
 52
 53static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0,
 54				 struct cppc_perf_fb_ctrs *fb_ctrs_t1);
 55
 56/**
 57 * cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance
 58 * @work: The work item.
 59 *
 60 * The CPPC driver register itself with the topology core to provide its own
 61 * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which
 62 * gets called by the scheduler on every tick.
 63 *
 64 * Note that the arch specific counters have higher priority than CPPC counters,
 65 * if available, though the CPPC driver doesn't need to have any special
 66 * handling for that.
 67 *
 68 * On an invocation of cppc_scale_freq_tick(), we schedule an irq work (since we
 69 * reach here from hard-irq context), which then schedules a normal work item
 70 * and cppc_scale_freq_workfn() updates the per_cpu arch_freq_scale variable
 71 * based on the counter updates since the last tick.
 72 */
 73static void cppc_scale_freq_workfn(struct kthread_work *work)
 74{
 75	struct cppc_freq_invariance *cppc_fi;
 76	struct cppc_perf_fb_ctrs fb_ctrs = {0};
 77	struct cppc_cpudata *cpu_data;
 78	unsigned long local_freq_scale;
 79	u64 perf;
 80
 81	cppc_fi = container_of(work, struct cppc_freq_invariance, work);
 82	cpu_data = cppc_fi->cpu_data;
 83
 84	if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) {
 85		pr_warn("%s: failed to read perf counters\n", __func__);
 86		return;
 87	}
 88
 89	perf = cppc_perf_from_fbctrs(&cppc_fi->prev_perf_fb_ctrs, &fb_ctrs);
 90	if (!perf)
 91		return;
 92
 93	cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
 94
 95	perf <<= SCHED_CAPACITY_SHIFT;
 96	local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf);
 97
 98	/* This can happen due to counter's overflow */
 99	if (unlikely(local_freq_scale > 1024))
100		local_freq_scale = 1024;
101
102	per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale;
103}
104
105static void cppc_irq_work(struct irq_work *irq_work)
106{
107	struct cppc_freq_invariance *cppc_fi;
108
109	cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work);
110	kthread_queue_work(kworker_fie, &cppc_fi->work);
111}
112
113static void cppc_scale_freq_tick(void)
114{
115	struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id());
116
117	/*
118	 * cppc_get_perf_ctrs() can potentially sleep, call that from the right
119	 * context.
120	 */
121	irq_work_queue(&cppc_fi->irq_work);
122}
123
124static struct scale_freq_data cppc_sftd = {
125	.source = SCALE_FREQ_SOURCE_CPPC,
126	.set_freq_scale = cppc_scale_freq_tick,
127};
128
129static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
130{
131	struct cppc_freq_invariance *cppc_fi;
132	int cpu, ret;
133
134	if (fie_disabled)
135		return;
136
137	for_each_cpu(cpu, policy->cpus) {
138		cppc_fi = &per_cpu(cppc_freq_inv, cpu);
139		cppc_fi->cpu = cpu;
140		cppc_fi->cpu_data = policy->driver_data;
141		kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn);
142		init_irq_work(&cppc_fi->irq_work, cppc_irq_work);
143
144		ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs);
145
146		/*
147		 * Don't abort as the CPU was offline while the driver was
148		 * getting registered.
149		 */
150		if (ret && cpu_online(cpu)) {
151			pr_debug("%s: failed to read perf counters for cpu:%d: %d\n",
152				__func__, cpu, ret);
153			return;
154		}
155	}
156
157	/* Register for freq-invariance */
158	topology_set_scale_freq_source(&cppc_sftd, policy->cpus);
159}
160
161/*
162 * We free all the resources on policy's removal and not on CPU removal as the
163 * irq-work are per-cpu and the hotplug core takes care of flushing the pending
164 * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work
165 * fires on another CPU after the concerned CPU is removed, it won't harm.
166 *
167 * We just need to make sure to remove them all on policy->exit().
168 */
169static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
170{
171	struct cppc_freq_invariance *cppc_fi;
172	int cpu;
173
174	if (fie_disabled)
175		return;
176
177	/* policy->cpus will be empty here, use related_cpus instead */
178	topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus);
179
180	for_each_cpu(cpu, policy->related_cpus) {
181		cppc_fi = &per_cpu(cppc_freq_inv, cpu);
182		irq_work_sync(&cppc_fi->irq_work);
183		kthread_cancel_work_sync(&cppc_fi->work);
184	}
185}
186
187static void __init cppc_freq_invariance_init(void)
188{
189	struct sched_attr attr = {
190		.size		= sizeof(struct sched_attr),
191		.sched_policy	= SCHED_DEADLINE,
192		.sched_nice	= 0,
193		.sched_priority	= 0,
194		/*
195		 * Fake (unused) bandwidth; workaround to "fix"
196		 * priority inheritance.
197		 */
198		.sched_runtime	= NSEC_PER_MSEC,
199		.sched_deadline = 10 * NSEC_PER_MSEC,
200		.sched_period	= 10 * NSEC_PER_MSEC,
201	};
202	int ret;
203
204	if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) {
205		fie_disabled = FIE_ENABLED;
206		if (cppc_perf_ctrs_in_pcc()) {
207			pr_info("FIE not enabled on systems with registers in PCC\n");
208			fie_disabled = FIE_DISABLED;
209		}
210	}
211
212	if (fie_disabled)
213		return;
214
215	kworker_fie = kthread_run_worker(0, "cppc_fie");
216	if (IS_ERR(kworker_fie)) {
217		pr_warn("%s: failed to create kworker_fie: %ld\n", __func__,
218			PTR_ERR(kworker_fie));
219		fie_disabled = FIE_DISABLED;
220		return;
221	}
222
223	ret = sched_setattr_nocheck(kworker_fie->task, &attr);
224	if (ret) {
225		pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__,
226			ret);
227		kthread_destroy_worker(kworker_fie);
228		fie_disabled = FIE_DISABLED;
229	}
230}
231
232static void cppc_freq_invariance_exit(void)
233{
234	if (fie_disabled)
235		return;
236
237	kthread_destroy_worker(kworker_fie);
238}
239
240#else
241static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
242{
243}
244
245static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
246{
247}
248
249static inline void cppc_freq_invariance_init(void)
250{
251}
252
253static inline void cppc_freq_invariance_exit(void)
254{
255}
256#endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */
257
258static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
259				   unsigned int target_freq,
260				   unsigned int relation)
261{
262	struct cppc_cpudata *cpu_data = policy->driver_data;
263	unsigned int cpu = policy->cpu;
264	struct cpufreq_freqs freqs;
265	int ret = 0;
266
267	cpu_data->perf_ctrls.desired_perf =
268			cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);
269	freqs.old = policy->cur;
270	freqs.new = target_freq;
271
272	cpufreq_freq_transition_begin(policy, &freqs);
273	ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
274	cpufreq_freq_transition_end(policy, &freqs, ret != 0);
275
276	if (ret)
277		pr_debug("Failed to set target on CPU:%d. ret:%d\n",
278			 cpu, ret);
279
280	return ret;
281}
282
283static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy,
284					      unsigned int target_freq)
285{
286	struct cppc_cpudata *cpu_data = policy->driver_data;
287	unsigned int cpu = policy->cpu;
288	u32 desired_perf;
289	int ret;
290
291	desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);
292	cpu_data->perf_ctrls.desired_perf = desired_perf;
293	ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
294
295	if (ret) {
296		pr_debug("Failed to set target on CPU:%d. ret:%d\n",
297			 cpu, ret);
298		return 0;
299	}
300
301	return target_freq;
302}
303
304static int cppc_verify_policy(struct cpufreq_policy_data *policy)
305{
306	cpufreq_verify_within_cpu_limits(policy);
307	return 0;
308}
309
310static unsigned int __cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
311{
312	int transition_latency_ns = cppc_get_transition_latency(cpu);
313
314	if (transition_latency_ns < 0)
315		return CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS / NSEC_PER_USEC;
316
317	return transition_latency_ns / NSEC_PER_USEC;
318}
319
320/*
321 * The PCC subspace describes the rate at which platform can accept commands
322 * on the shared PCC channel (including READs which do not count towards freq
323 * transition requests), so ideally we need to use the PCC values as a fallback
324 * if we don't have a platform specific transition_delay_us
325 */
326#ifdef CONFIG_ARM64
327#include <asm/cputype.h>
328
329static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
330{
331	unsigned long implementor = read_cpuid_implementor();
332	unsigned long part_num = read_cpuid_part_number();
333
334	switch (implementor) {
335	case ARM_CPU_IMP_QCOM:
336		switch (part_num) {
337		case QCOM_CPU_PART_FALKOR_V1:
338		case QCOM_CPU_PART_FALKOR:
339			return 10000;
340		}
341	}
342	return __cppc_cpufreq_get_transition_delay_us(cpu);
343}
344#else
345static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
346{
347	return __cppc_cpufreq_get_transition_delay_us(cpu);
348}
349#endif
350
351#if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL)
352
353static DEFINE_PER_CPU(unsigned int, efficiency_class);
354
355/* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */
356#define CPPC_EM_CAP_STEP	(20)
357/* Increase the cost value by CPPC_EM_COST_STEP every performance state. */
358#define CPPC_EM_COST_STEP	(1)
359/* Add a cost gap correspnding to the energy of 4 CPUs. */
360#define CPPC_EM_COST_GAP	(4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \
361				/ CPPC_EM_CAP_STEP)
362
363static unsigned int get_perf_level_count(struct cpufreq_policy *policy)
364{
365	struct cppc_perf_caps *perf_caps;
366	unsigned int min_cap, max_cap;
367	struct cppc_cpudata *cpu_data;
368	int cpu = policy->cpu;
369
370	cpu_data = policy->driver_data;
371	perf_caps = &cpu_data->perf_caps;
372	max_cap = arch_scale_cpu_capacity(cpu);
373	min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,
374			  perf_caps->highest_perf);
375	if ((min_cap == 0) || (max_cap < min_cap))
376		return 0;
377	return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP;
378}
379
380/*
381 * The cost is defined as:
382 *   cost = power * max_frequency / frequency
383 */
384static inline unsigned long compute_cost(int cpu, int step)
385{
386	return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) +
387			step * CPPC_EM_COST_STEP;
388}
389
390static int cppc_get_cpu_power(struct device *cpu_dev,
391		unsigned long *power, unsigned long *KHz)
392{
393	unsigned long perf_step, perf_prev, perf, perf_check;
394	unsigned int min_step, max_step, step, step_check;
395	unsigned long prev_freq = *KHz;
396	unsigned int min_cap, max_cap;
397	struct cpufreq_policy *policy;
398
399	struct cppc_perf_caps *perf_caps;
400	struct cppc_cpudata *cpu_data;
401
402	policy = cpufreq_cpu_get_raw(cpu_dev->id);
403	if (!policy)
404		return -EINVAL;
405
406	cpu_data = policy->driver_data;
407	perf_caps = &cpu_data->perf_caps;
408	max_cap = arch_scale_cpu_capacity(cpu_dev->id);
409	min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,
410			  perf_caps->highest_perf);
411	perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf,
412			    max_cap);
413	min_step = min_cap / CPPC_EM_CAP_STEP;
414	max_step = max_cap / CPPC_EM_CAP_STEP;
415
416	perf_prev = cppc_khz_to_perf(perf_caps, *KHz);
417	step = perf_prev / perf_step;
418
419	if (step > max_step)
420		return -EINVAL;
421
422	if (min_step == max_step) {
423		step = max_step;
424		perf = perf_caps->highest_perf;
425	} else if (step < min_step) {
426		step = min_step;
427		perf = perf_caps->lowest_perf;
428	} else {
429		step++;
430		if (step == max_step)
431			perf = perf_caps->highest_perf;
432		else
433			perf = step * perf_step;
434	}
435
436	*KHz = cppc_perf_to_khz(perf_caps, perf);
437	perf_check = cppc_khz_to_perf(perf_caps, *KHz);
438	step_check = perf_check / perf_step;
439
440	/*
441	 * To avoid bad integer approximation, check that new frequency value
442	 * increased and that the new frequency will be converted to the
443	 * desired step value.
444	 */
445	while ((*KHz == prev_freq) || (step_check != step)) {
446		perf++;
447		*KHz = cppc_perf_to_khz(perf_caps, perf);
448		perf_check = cppc_khz_to_perf(perf_caps, *KHz);
449		step_check = perf_check / perf_step;
450	}
451
452	/*
453	 * With an artificial EM, only the cost value is used. Still the power
454	 * is populated such as 0 < power < EM_MAX_POWER. This allows to add
455	 * more sense to the artificial performance states.
456	 */
457	*power = compute_cost(cpu_dev->id, step);
458
459	return 0;
460}
461
462static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz,
463		unsigned long *cost)
464{
465	unsigned long perf_step, perf_prev;
466	struct cppc_perf_caps *perf_caps;
467	struct cpufreq_policy *policy;
468	struct cppc_cpudata *cpu_data;
469	unsigned int max_cap;
470	int step;
471
472	policy = cpufreq_cpu_get_raw(cpu_dev->id);
473	if (!policy)
474		return -EINVAL;
475
476	cpu_data = policy->driver_data;
477	perf_caps = &cpu_data->perf_caps;
478	max_cap = arch_scale_cpu_capacity(cpu_dev->id);
479
480	perf_prev = cppc_khz_to_perf(perf_caps, KHz);
481	perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap;
482	step = perf_prev / perf_step;
483
484	*cost = compute_cost(cpu_dev->id, step);
485
486	return 0;
487}
488
489static void cppc_cpufreq_register_em(struct cpufreq_policy *policy)
490{
491	struct cppc_cpudata *cpu_data;
492	struct em_data_callback em_cb =
493		EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost);
494
495	cpu_data = policy->driver_data;
496	em_dev_register_perf_domain(get_cpu_device(policy->cpu),
497			get_perf_level_count(policy), &em_cb,
498			cpu_data->shared_cpu_map, 0);
499}
500
501static void populate_efficiency_class(void)
502{
503	struct acpi_madt_generic_interrupt *gicc;
504	DECLARE_BITMAP(used_classes, 256) = {};
505	int class, cpu, index;
506
507	for_each_possible_cpu(cpu) {
508		gicc = acpi_cpu_get_madt_gicc(cpu);
509		class = gicc->efficiency_class;
510		bitmap_set(used_classes, class, 1);
511	}
512
513	if (bitmap_weight(used_classes, 256) <= 1) {
514		pr_debug("Efficiency classes are all equal (=%d). "
515			"No EM registered", class);
516		return;
517	}
518
519	/*
520	 * Squeeze efficiency class values on [0:#efficiency_class-1].
521	 * Values are per spec in [0:255].
522	 */
523	index = 0;
524	for_each_set_bit(class, used_classes, 256) {
525		for_each_possible_cpu(cpu) {
526			gicc = acpi_cpu_get_madt_gicc(cpu);
527			if (gicc->efficiency_class == class)
528				per_cpu(efficiency_class, cpu) = index;
529		}
530		index++;
531	}
532	cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em;
533}
534
535#else
536static void populate_efficiency_class(void)
537{
538}
539#endif
540
541static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu)
542{
543	struct cppc_cpudata *cpu_data;
544	int ret;
545
546	cpu_data = kzalloc(sizeof(struct cppc_cpudata), GFP_KERNEL);
547	if (!cpu_data)
548		goto out;
549
550	if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL))
551		goto free_cpu;
552
553	ret = acpi_get_psd_map(cpu, cpu_data);
554	if (ret) {
555		pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret);
556		goto free_mask;
557	}
558
559	ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps);
560	if (ret) {
561		pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret);
562		goto free_mask;
563	}
564
565	return cpu_data;
566
567free_mask:
568	free_cpumask_var(cpu_data->shared_cpu_map);
569free_cpu:
570	kfree(cpu_data);
571out:
572	return NULL;
573}
574
575static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy)
576{
577	struct cppc_cpudata *cpu_data = policy->driver_data;
578
579	free_cpumask_var(cpu_data->shared_cpu_map);
580	kfree(cpu_data);
581	policy->driver_data = NULL;
582}
583
584static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
585{
586	unsigned int cpu = policy->cpu;
587	struct cppc_cpudata *cpu_data;
588	struct cppc_perf_caps *caps;
589	int ret;
590
591	cpu_data = cppc_cpufreq_get_cpu_data(cpu);
592	if (!cpu_data) {
593		pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu);
594		return -ENODEV;
595	}
596	caps = &cpu_data->perf_caps;
597	policy->driver_data = cpu_data;
598
599	/*
600	 * Set min to lowest nonlinear perf to avoid any efficiency penalty (see
601	 * Section 8.4.7.1.1.5 of ACPI 6.1 spec)
602	 */
603	policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf);
604	policy->max = cppc_perf_to_khz(caps, policy->boost_enabled ?
605						caps->highest_perf : caps->nominal_perf);
606
607	/*
608	 * Set cpuinfo.min_freq to Lowest to make the full range of performance
609	 * available if userspace wants to use any perf between lowest & lowest
610	 * nonlinear perf
611	 */
612	policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf);
613	policy->cpuinfo.max_freq = policy->max;
614
615	policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu);
616	policy->shared_type = cpu_data->shared_type;
617
618	switch (policy->shared_type) {
619	case CPUFREQ_SHARED_TYPE_HW:
620	case CPUFREQ_SHARED_TYPE_NONE:
621		/* Nothing to be done - we'll have a policy for each CPU */
622		break;
623	case CPUFREQ_SHARED_TYPE_ANY:
624		/*
625		 * All CPUs in the domain will share a policy and all cpufreq
626		 * operations will use a single cppc_cpudata structure stored
627		 * in policy->driver_data.
628		 */
629		cpumask_copy(policy->cpus, cpu_data->shared_cpu_map);
630		break;
631	default:
632		pr_debug("Unsupported CPU co-ord type: %d\n",
633			 policy->shared_type);
634		ret = -EFAULT;
635		goto out;
636	}
637
638	policy->fast_switch_possible = cppc_allow_fast_switch();
639	policy->dvfs_possible_from_any_cpu = true;
640
641	/*
642	 * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost
643	 * is supported.
644	 */
645	if (caps->highest_perf > caps->nominal_perf)
646		policy->boost_supported = true;
647
648	/* Set policy->cur to max now. The governors will adjust later. */
649	policy->cur = cppc_perf_to_khz(caps, caps->highest_perf);
650	cpu_data->perf_ctrls.desired_perf =  caps->highest_perf;
651
652	ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
653	if (ret) {
654		pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
655			 caps->highest_perf, cpu, ret);
656		goto out;
657	}
658
659	cppc_cpufreq_cpu_fie_init(policy);
660	return 0;
661
662out:
663	cppc_cpufreq_put_cpu_data(policy);
664	return ret;
665}
666
667static void cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy)
668{
669	struct cppc_cpudata *cpu_data = policy->driver_data;
670	struct cppc_perf_caps *caps = &cpu_data->perf_caps;
671	unsigned int cpu = policy->cpu;
672	int ret;
673
674	cppc_cpufreq_cpu_fie_exit(policy);
675
676	cpu_data->perf_ctrls.desired_perf = caps->lowest_perf;
677
678	ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
679	if (ret)
680		pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
681			 caps->lowest_perf, cpu, ret);
682
683	cppc_cpufreq_put_cpu_data(policy);
684}
685
686static inline u64 get_delta(u64 t1, u64 t0)
687{
688	if (t1 > t0 || t0 > ~(u32)0)
689		return t1 - t0;
690
691	return (u32)t1 - (u32)t0;
692}
693
694static int cppc_perf_from_fbctrs(struct cppc_perf_fb_ctrs *fb_ctrs_t0,
695				 struct cppc_perf_fb_ctrs *fb_ctrs_t1)
696{
697	u64 delta_reference, delta_delivered;
698	u64 reference_perf;
699
700	reference_perf = fb_ctrs_t0->reference_perf;
701
702	delta_reference = get_delta(fb_ctrs_t1->reference,
703				    fb_ctrs_t0->reference);
704	delta_delivered = get_delta(fb_ctrs_t1->delivered,
705				    fb_ctrs_t0->delivered);
706
707	/*
708	 * Avoid divide-by zero and unchanged feedback counters.
709	 * Leave it for callers to handle.
710	 */
711	if (!delta_reference || !delta_delivered)
712		return 0;
713
714	return (reference_perf * delta_delivered) / delta_reference;
715}
716
717static int cppc_get_perf_ctrs_sample(int cpu,
718				     struct cppc_perf_fb_ctrs *fb_ctrs_t0,
719				     struct cppc_perf_fb_ctrs *fb_ctrs_t1)
720{
721	int ret;
722
723	ret = cppc_get_perf_ctrs(cpu, fb_ctrs_t0);
724	if (ret)
725		return ret;
726
727	udelay(2); /* 2usec delay between sampling */
728
729	return cppc_get_perf_ctrs(cpu, fb_ctrs_t1);
730}
731
732static unsigned int cppc_cpufreq_get_rate(unsigned int cpu)
733{
734	struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
735	struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0};
736	struct cppc_cpudata *cpu_data;
737	u64 delivered_perf;
738	int ret;
739
740	if (!policy)
741		return 0;
742
743	cpu_data = policy->driver_data;
744
745	ret = cppc_get_perf_ctrs_sample(cpu, &fb_ctrs_t0, &fb_ctrs_t1);
746	if (ret) {
747		if (ret == -EFAULT)
748			/* Any of the associated CPPC regs is 0. */
749			goto out_invalid_counters;
750		else
751			return 0;
752	}
753
754	delivered_perf = cppc_perf_from_fbctrs(&fb_ctrs_t0, &fb_ctrs_t1);
755	if (!delivered_perf)
756		goto out_invalid_counters;
757
758	return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf);
759
760out_invalid_counters:
761	/*
762	 * Feedback counters could be unchanged or 0 when a cpu enters a
763	 * low-power idle state, e.g. clock-gated or power-gated.
764	 * Use desired perf for reflecting frequency.  Get the latest register
765	 * value first as some platforms may update the actual delivered perf
766	 * there; if failed, resort to the cached desired perf.
767	 */
768	if (cppc_get_desired_perf(cpu, &delivered_perf))
769		delivered_perf = cpu_data->perf_ctrls.desired_perf;
770
771	return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf);
772}
773
774static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
775{
776	struct cppc_cpudata *cpu_data = policy->driver_data;
777	struct cppc_perf_caps *caps = &cpu_data->perf_caps;
778	int ret;
779
780	if (state)
781		policy->max = cppc_perf_to_khz(caps, caps->highest_perf);
782	else
783		policy->max = cppc_perf_to_khz(caps, caps->nominal_perf);
784	policy->cpuinfo.max_freq = policy->max;
785
786	ret = freq_qos_update_request(policy->max_freq_req, policy->max);
787	if (ret < 0)
788		return ret;
789
790	return 0;
791}
792
793static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)
794{
795	struct cppc_cpudata *cpu_data = policy->driver_data;
796
797	return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf);
798}
799
800static ssize_t show_auto_select(struct cpufreq_policy *policy, char *buf)
801{
802	bool val;
803	int ret;
804
805	ret = cppc_get_auto_sel(policy->cpu, &val);
806
807	/* show "<unsupported>" when this register is not supported by cpc */
808	if (ret == -EOPNOTSUPP)
809		return sysfs_emit(buf, "<unsupported>\n");
810
811	if (ret)
812		return ret;
813
814	return sysfs_emit(buf, "%d\n", val);
815}
816
817static ssize_t store_auto_select(struct cpufreq_policy *policy,
818				 const char *buf, size_t count)
819{
820	bool val;
821	int ret;
822
823	ret = kstrtobool(buf, &val);
824	if (ret)
825		return ret;
826
827	ret = cppc_set_auto_sel(policy->cpu, val);
828	if (ret)
829		return ret;
830
831	return count;
832}
833
834static ssize_t show_auto_act_window(struct cpufreq_policy *policy, char *buf)
835{
836	u64 val;
837	int ret;
838
839	ret = cppc_get_auto_act_window(policy->cpu, &val);
840
841	/* show "<unsupported>" when this register is not supported by cpc */
842	if (ret == -EOPNOTSUPP)
843		return sysfs_emit(buf, "<unsupported>\n");
844
845	if (ret)
846		return ret;
847
848	return sysfs_emit(buf, "%llu\n", val);
849}
850
851static ssize_t store_auto_act_window(struct cpufreq_policy *policy,
852				     const char *buf, size_t count)
853{
854	u64 usec;
855	int ret;
856
857	ret = kstrtou64(buf, 0, &usec);
858	if (ret)
859		return ret;
860
861	ret = cppc_set_auto_act_window(policy->cpu, usec);
862	if (ret)
863		return ret;
864
865	return count;
866}
867
868static ssize_t show_energy_performance_preference_val(struct cpufreq_policy *policy, char *buf)
869{
870	u64 val;
871	int ret;
872
873	ret = cppc_get_epp_perf(policy->cpu, &val);
874
875	/* show "<unsupported>" when this register is not supported by cpc */
876	if (ret == -EOPNOTSUPP)
877		return sysfs_emit(buf, "<unsupported>\n");
878
879	if (ret)
880		return ret;
881
882	return sysfs_emit(buf, "%llu\n", val);
883}
884
885static ssize_t store_energy_performance_preference_val(struct cpufreq_policy *policy,
886						       const char *buf, size_t count)
887{
888	u64 val;
889	int ret;
890
891	ret = kstrtou64(buf, 0, &val);
892	if (ret)
893		return ret;
894
895	ret = cppc_set_epp(policy->cpu, val);
896	if (ret)
897		return ret;
898
899	return count;
900}
901
902cpufreq_freq_attr_ro(freqdomain_cpus);
903cpufreq_freq_attr_rw(auto_select);
904cpufreq_freq_attr_rw(auto_act_window);
905cpufreq_freq_attr_rw(energy_performance_preference_val);
906
907static struct freq_attr *cppc_cpufreq_attr[] = {
908	&freqdomain_cpus,
909	&auto_select,
910	&auto_act_window,
911	&energy_performance_preference_val,
912	NULL,
913};
914
915static struct cpufreq_driver cppc_cpufreq_driver = {
916	.flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
917	.verify = cppc_verify_policy,
918	.target = cppc_cpufreq_set_target,
919	.get = cppc_cpufreq_get_rate,
920	.fast_switch = cppc_cpufreq_fast_switch,
921	.init = cppc_cpufreq_cpu_init,
922	.exit = cppc_cpufreq_cpu_exit,
923	.set_boost = cppc_cpufreq_set_boost,
924	.attr = cppc_cpufreq_attr,
925	.name = "cppc_cpufreq",
926};
927
928static int __init cppc_cpufreq_init(void)
929{
930	int ret;
931
932	if (!acpi_cpc_valid())
933		return -ENODEV;
934
935	cppc_freq_invariance_init();
936	populate_efficiency_class();
937
938	ret = cpufreq_register_driver(&cppc_cpufreq_driver);
939	if (ret)
940		cppc_freq_invariance_exit();
941
942	return ret;
943}
944
945static void __exit cppc_cpufreq_exit(void)
946{
947	cpufreq_unregister_driver(&cppc_cpufreq_driver);
948	cppc_freq_invariance_exit();
949}
950
951module_exit(cppc_cpufreq_exit);
952MODULE_AUTHOR("Ashwin Chaugule");
953MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec");
954MODULE_LICENSE("GPL");
955
956late_initcall(cppc_cpufreq_init);
957
958static const struct acpi_device_id cppc_acpi_ids[] __used = {
959	{ACPI_PROCESSOR_DEVICE_HID, },
960	{}
961};
962
963MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids);