tegra30-devfreq.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * A devfreq driver for NVIDIA Tegra SoCs
  4 *
  5 * Copyright (c) 2014 NVIDIA CORPORATION. All rights reserved.
  6 * Copyright (C) 2014 Google, Inc
  7 */
  8
  9#include <linux/clk.h>
 10#include <linux/cpufreq.h>
 11#include <linux/devfreq.h>
 12#include <linux/devfreq-governor.h>
 13#include <linux/interrupt.h>
 14#include <linux/io.h>
 15#include <linux/irq.h>
 16#include <linux/minmax.h>
 17#include <linux/module.h>
 18#include <linux/of.h>
 19#include <linux/platform_device.h>
 20#include <linux/pm_opp.h>
 21#include <linux/reset.h>
 22#include <linux/workqueue.h>
 23
 24#include <soc/tegra/fuse.h>
 25
 26#define ACTMON_GLB_STATUS					0x0
 27#define ACTMON_GLB_PERIOD_CTRL					0x4
 28
 29#define ACTMON_DEV_CTRL						0x0
 30#define ACTMON_DEV_CTRL_K_VAL_SHIFT				10
 31#define ACTMON_DEV_CTRL_ENB_PERIODIC				BIT(18)
 32#define ACTMON_DEV_CTRL_AVG_BELOW_WMARK_EN			BIT(20)
 33#define ACTMON_DEV_CTRL_AVG_ABOVE_WMARK_EN			BIT(21)
 34#define ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_NUM_SHIFT	23
 35#define ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_NUM_SHIFT	26
 36#define ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN		BIT(29)
 37#define ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN		BIT(30)
 38#define ACTMON_DEV_CTRL_ENB					BIT(31)
 39
 40#define ACTMON_DEV_CTRL_STOP					0x00000000
 41
 42#define ACTMON_DEV_UPPER_WMARK					0x4
 43#define ACTMON_DEV_LOWER_WMARK					0x8
 44#define ACTMON_DEV_INIT_AVG					0xc
 45#define ACTMON_DEV_AVG_UPPER_WMARK				0x10
 46#define ACTMON_DEV_AVG_LOWER_WMARK				0x14
 47#define ACTMON_DEV_COUNT_WEIGHT					0x18
 48#define ACTMON_DEV_AVG_COUNT					0x20
 49#define ACTMON_DEV_INTR_STATUS					0x24
 50
 51#define ACTMON_INTR_STATUS_CLEAR				0xffffffff
 52
 53#define ACTMON_DEV_INTR_CONSECUTIVE_UPPER			BIT(31)
 54#define ACTMON_DEV_INTR_CONSECUTIVE_LOWER			BIT(30)
 55
 56#define ACTMON_ABOVE_WMARK_WINDOW				1
 57#define ACTMON_BELOW_WMARK_WINDOW				3
 58#define ACTMON_BOOST_FREQ_STEP					16000
 59
 60/*
 61 * ACTMON_AVERAGE_WINDOW_LOG2: default value for @DEV_CTRL_K_VAL, which
 62 * translates to 2 ^ (K_VAL + 1). ex: 2 ^ (6 + 1) = 128
 63 */
 64#define ACTMON_AVERAGE_WINDOW_LOG2			6
 65#define ACTMON_SAMPLING_PERIOD				12 /* ms */
 66#define ACTMON_DEFAULT_AVG_BAND				6  /* 1/10 of % */
 67
 68#define KHZ							1000
 69
 70#define KHZ_MAX						(ULONG_MAX / KHZ)
 71
 72/* Assume that the bus is saturated if the utilization is 25% */
 73#define BUS_SATURATION_RATIO					25
 74
 75/**
 76 * struct tegra_devfreq_device_config - configuration specific to an ACTMON
 77 * device
 78 *
 79 * Coefficients and thresholds are percentages unless otherwise noted
 80 */
 81struct tegra_devfreq_device_config {
 82	u32		offset;
 83	u32		irq_mask;
 84
 85	/* Factors applied to boost_freq every consecutive watermark breach */
 86	unsigned int	boost_up_coeff;
 87	unsigned int	boost_down_coeff;
 88
 89	/* Define the watermark bounds when applied to the current avg */
 90	unsigned int	boost_up_threshold;
 91	unsigned int	boost_down_threshold;
 92
 93	/*
 94	 * Threshold of activity (cycles translated to kHz) below which the
 95	 * CPU frequency isn't to be taken into account. This is to avoid
 96	 * increasing the EMC frequency when the CPU is very busy but not
 97	 * accessing the bus often.
 98	 */
 99	u32		avg_dependency_threshold;
100};
101
102enum tegra_actmon_device {
103	MCALL = 0,
104	MCCPU,
105};
106
107static const struct tegra_devfreq_device_config tegra124_device_configs[] = {
108	{
109		/* MCALL: All memory accesses (including from the CPUs) */
110		.offset = 0x1c0,
111		.irq_mask = 1 << 26,
112		.boost_up_coeff = 200,
113		.boost_down_coeff = 50,
114		.boost_up_threshold = 60,
115		.boost_down_threshold = 40,
116	},
117	{
118		/* MCCPU: memory accesses from the CPUs */
119		.offset = 0x200,
120		.irq_mask = 1 << 25,
121		.boost_up_coeff = 800,
122		.boost_down_coeff = 40,
123		.boost_up_threshold = 27,
124		.boost_down_threshold = 10,
125		.avg_dependency_threshold = 16000, /* 16MHz in kHz units */
126	},
127};
128
129static const struct tegra_devfreq_device_config tegra30_device_configs[] = {
130	{
131		/* MCALL: All memory accesses (including from the CPUs) */
132		.offset = 0x1c0,
133		.irq_mask = 1 << 26,
134		.boost_up_coeff = 200,
135		.boost_down_coeff = 50,
136		.boost_up_threshold = 20,
137		.boost_down_threshold = 10,
138	},
139	{
140		/* MCCPU: memory accesses from the CPUs */
141		.offset = 0x200,
142		.irq_mask = 1 << 25,
143		.boost_up_coeff = 800,
144		.boost_down_coeff = 40,
145		.boost_up_threshold = 27,
146		.boost_down_threshold = 10,
147		.avg_dependency_threshold = 16000, /* 16MHz in kHz units */
148	},
149};
150
151/**
152 * struct tegra_devfreq_device - state specific to an ACTMON device
153 *
154 * Frequencies are in kHz.
155 */
156struct tegra_devfreq_device {
157	const struct tegra_devfreq_device_config *config;
158	void __iomem *regs;
159
160	/* Average event count sampled in the last interrupt */
161	u32 avg_count;
162
163	/*
164	 * Extra frequency to increase the target by due to consecutive
165	 * watermark breaches.
166	 */
167	unsigned long boost_freq;
168
169	/* Optimal frequency calculated from the stats for this device */
170	unsigned long target_freq;
171};
172
173struct tegra_devfreq_soc_data {
174	const struct tegra_devfreq_device_config *configs;
175	/* Weight value for count measurements */
176	unsigned int count_weight;
177};
178
179struct tegra_devfreq {
180	struct devfreq		*devfreq;
181
182	struct reset_control	*reset;
183	struct clk		*clock;
184	void __iomem		*regs;
185
186	struct clk		*emc_clock;
187	unsigned long		max_freq;
188	unsigned long		cur_freq;
189	struct notifier_block	clk_rate_change_nb;
190
191	struct delayed_work	cpufreq_update_work;
192	struct notifier_block	cpu_rate_change_nb;
193
194	struct tegra_devfreq_device devices[2];
195
196	unsigned int		irq;
197
198	bool			started;
199
200	const struct tegra_devfreq_soc_data *soc;
201};
202
203struct tegra_actmon_emc_ratio {
204	unsigned long cpu_freq;
205	unsigned long emc_freq;
206};
207
208static const struct tegra_actmon_emc_ratio actmon_emc_ratios[] = {
209	{ 1400000,    KHZ_MAX },
210	{ 1200000,    750000 },
211	{ 1100000,    600000 },
212	{ 1000000,    500000 },
213	{  800000,    375000 },
214	{  500000,    200000 },
215	{  250000,    100000 },
216};
217
218static u32 actmon_readl(struct tegra_devfreq *tegra, u32 offset)
219{
220	return readl_relaxed(tegra->regs + offset);
221}
222
223static void actmon_writel(struct tegra_devfreq *tegra, u32 val, u32 offset)
224{
225	writel_relaxed(val, tegra->regs + offset);
226}
227
228static u32 device_readl(struct tegra_devfreq_device *dev, u32 offset)
229{
230	return readl_relaxed(dev->regs + offset);
231}
232
233static void device_writel(struct tegra_devfreq_device *dev, u32 val,
234			  u32 offset)
235{
236	writel_relaxed(val, dev->regs + offset);
237}
238
239static unsigned long do_percent(unsigned long long val, unsigned int pct)
240{
241	val = val * pct;
242	do_div(val, 100);
243
244	/*
245	 * High freq + high boosting percent + large polling interval are
246	 * resulting in integer overflow when watermarks are calculated.
247	 */
248	return min_t(u64, val, U32_MAX);
249}
250
251static void tegra_devfreq_update_avg_wmark(struct tegra_devfreq *tegra,
252					   struct tegra_devfreq_device *dev)
253{
254	u32 avg_band_freq = tegra->max_freq * ACTMON_DEFAULT_AVG_BAND / KHZ;
255	u32 band = avg_band_freq * tegra->devfreq->profile->polling_ms;
256	u32 avg;
257
258	avg = min(dev->avg_count, U32_MAX - band);
259	device_writel(dev, avg + band, ACTMON_DEV_AVG_UPPER_WMARK);
260
261	avg = max(dev->avg_count, band);
262	device_writel(dev, avg - band, ACTMON_DEV_AVG_LOWER_WMARK);
263}
264
265static void tegra_devfreq_update_wmark(struct tegra_devfreq *tegra,
266				       struct tegra_devfreq_device *dev)
267{
268	u32 val = tegra->cur_freq * tegra->devfreq->profile->polling_ms;
269
270	device_writel(dev, do_percent(val, dev->config->boost_up_threshold),
271		      ACTMON_DEV_UPPER_WMARK);
272
273	device_writel(dev, do_percent(val, dev->config->boost_down_threshold),
274		      ACTMON_DEV_LOWER_WMARK);
275}
276
277static void actmon_isr_device(struct tegra_devfreq *tegra,
278			      struct tegra_devfreq_device *dev)
279{
280	u32 intr_status, dev_ctrl;
281
282	dev->avg_count = device_readl(dev, ACTMON_DEV_AVG_COUNT);
283	tegra_devfreq_update_avg_wmark(tegra, dev);
284
285	intr_status = device_readl(dev, ACTMON_DEV_INTR_STATUS);
286	dev_ctrl = device_readl(dev, ACTMON_DEV_CTRL);
287
288	if (intr_status & ACTMON_DEV_INTR_CONSECUTIVE_UPPER) {
289		/*
290		 * new_boost = min(old_boost * up_coef + step, max_freq)
291		 */
292		dev->boost_freq = do_percent(dev->boost_freq,
293					     dev->config->boost_up_coeff);
294		dev->boost_freq += ACTMON_BOOST_FREQ_STEP;
295
296		dev_ctrl |= ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN;
297
298		if (dev->boost_freq >= tegra->max_freq) {
299			dev_ctrl &= ~ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN;
300			dev->boost_freq = tegra->max_freq;
301		}
302	} else if (intr_status & ACTMON_DEV_INTR_CONSECUTIVE_LOWER) {
303		/*
304		 * new_boost = old_boost * down_coef
305		 * or 0 if (old_boost * down_coef < step / 2)
306		 */
307		dev->boost_freq = do_percent(dev->boost_freq,
308					     dev->config->boost_down_coeff);
309
310		dev_ctrl |= ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN;
311
312		if (dev->boost_freq < (ACTMON_BOOST_FREQ_STEP >> 1)) {
313			dev_ctrl &= ~ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN;
314			dev->boost_freq = 0;
315		}
316	}
317
318	device_writel(dev, dev_ctrl, ACTMON_DEV_CTRL);
319
320	device_writel(dev, ACTMON_INTR_STATUS_CLEAR, ACTMON_DEV_INTR_STATUS);
321}
322
323static unsigned long actmon_cpu_to_emc_rate(struct tegra_devfreq *tegra,
324					    unsigned long cpu_freq)
325{
326	unsigned int i;
327	const struct tegra_actmon_emc_ratio *ratio = actmon_emc_ratios;
328
329	for (i = 0; i < ARRAY_SIZE(actmon_emc_ratios); i++, ratio++)
330		if (cpu_freq >= ratio->cpu_freq)
331			return min(ratio->emc_freq, tegra->max_freq);
332
333	return 0;
334}
335
336static unsigned long actmon_device_target_freq(struct tegra_devfreq *tegra,
337					       struct tegra_devfreq_device *dev)
338{
339	unsigned int avg_sustain_coef;
340	unsigned long target_freq;
341
342	target_freq = dev->avg_count / tegra->devfreq->profile->polling_ms;
343	avg_sustain_coef = 100 * 100 / dev->config->boost_up_threshold;
344	target_freq = do_percent(target_freq, avg_sustain_coef);
345
346	return target_freq;
347}
348
349static void actmon_update_target(struct tegra_devfreq *tegra,
350				 struct tegra_devfreq_device *dev)
351{
352	unsigned long cpu_freq = 0;
353	unsigned long static_cpu_emc_freq = 0;
354
355	dev->target_freq = actmon_device_target_freq(tegra, dev);
356
357	if (dev->config->avg_dependency_threshold &&
358	    dev->config->avg_dependency_threshold <= dev->target_freq) {
359		cpu_freq = cpufreq_quick_get(0);
360		static_cpu_emc_freq = actmon_cpu_to_emc_rate(tegra, cpu_freq);
361
362		dev->target_freq += dev->boost_freq;
363		dev->target_freq = max(dev->target_freq, static_cpu_emc_freq);
364	} else {
365		dev->target_freq += dev->boost_freq;
366	}
367}
368
369static irqreturn_t actmon_thread_isr(int irq, void *data)
370{
371	struct tegra_devfreq *tegra = data;
372	bool handled = false;
373	unsigned int i;
374	u32 val;
375
376	mutex_lock(&tegra->devfreq->lock);
377
378	val = actmon_readl(tegra, ACTMON_GLB_STATUS);
379	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
380		if (val & tegra->devices[i].config->irq_mask) {
381			actmon_isr_device(tegra, tegra->devices + i);
382			handled = true;
383		}
384	}
385
386	if (handled)
387		update_devfreq(tegra->devfreq);
388
389	mutex_unlock(&tegra->devfreq->lock);
390
391	return handled ? IRQ_HANDLED : IRQ_NONE;
392}
393
394static int tegra_actmon_clk_notify_cb(struct notifier_block *nb,
395				      unsigned long action, void *ptr)
396{
397	struct clk_notifier_data *data = ptr;
398	struct tegra_devfreq *tegra;
399	struct tegra_devfreq_device *dev;
400	unsigned int i;
401
402	if (action != POST_RATE_CHANGE)
403		return NOTIFY_OK;
404
405	tegra = container_of(nb, struct tegra_devfreq, clk_rate_change_nb);
406
407	tegra->cur_freq = data->new_rate / KHZ;
408
409	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
410		dev = &tegra->devices[i];
411
412		tegra_devfreq_update_wmark(tegra, dev);
413	}
414
415	return NOTIFY_OK;
416}
417
418static void tegra_actmon_delayed_update(struct work_struct *work)
419{
420	struct tegra_devfreq *tegra = container_of(work, struct tegra_devfreq,
421						   cpufreq_update_work.work);
422
423	mutex_lock(&tegra->devfreq->lock);
424	update_devfreq(tegra->devfreq);
425	mutex_unlock(&tegra->devfreq->lock);
426}
427
428static unsigned long
429tegra_actmon_cpufreq_contribution(struct tegra_devfreq *tegra,
430				  unsigned int cpu_freq)
431{
432	struct tegra_devfreq_device *actmon_dev = &tegra->devices[MCCPU];
433	unsigned long static_cpu_emc_freq, dev_freq;
434
435	dev_freq = actmon_device_target_freq(tegra, actmon_dev);
436
437	/* check whether CPU's freq is taken into account at all */
438	if (dev_freq < actmon_dev->config->avg_dependency_threshold)
439		return 0;
440
441	static_cpu_emc_freq = actmon_cpu_to_emc_rate(tegra, cpu_freq);
442
443	if (dev_freq + actmon_dev->boost_freq >= static_cpu_emc_freq)
444		return 0;
445
446	return static_cpu_emc_freq;
447}
448
449static int tegra_actmon_cpu_notify_cb(struct notifier_block *nb,
450				      unsigned long action, void *ptr)
451{
452	struct cpufreq_freqs *freqs = ptr;
453	struct tegra_devfreq *tegra;
454	unsigned long old, new, delay;
455
456	if (action != CPUFREQ_POSTCHANGE)
457		return NOTIFY_OK;
458
459	tegra = container_of(nb, struct tegra_devfreq, cpu_rate_change_nb);
460
461	/*
462	 * Quickly check whether CPU frequency should be taken into account
463	 * at all, without blocking CPUFreq's core.
464	 */
465	if (mutex_trylock(&tegra->devfreq->lock)) {
466		old = tegra_actmon_cpufreq_contribution(tegra, freqs->old);
467		new = tegra_actmon_cpufreq_contribution(tegra, freqs->new);
468		mutex_unlock(&tegra->devfreq->lock);
469
470		/*
471		 * If CPU's frequency shouldn't be taken into account at
472		 * the moment, then there is no need to update the devfreq's
473		 * state because ISR will re-check CPU's frequency on the
474		 * next interrupt.
475		 */
476		if (old == new)
477			return NOTIFY_OK;
478	}
479
480	/*
481	 * CPUFreq driver should support CPUFREQ_ASYNC_NOTIFICATION in order
482	 * to allow asynchronous notifications. This means we can't block
483	 * here for too long, otherwise CPUFreq's core will complain with a
484	 * warning splat.
485	 */
486	delay = msecs_to_jiffies(ACTMON_SAMPLING_PERIOD);
487	schedule_delayed_work(&tegra->cpufreq_update_work, delay);
488
489	return NOTIFY_OK;
490}
491
492static void tegra_actmon_configure_device(struct tegra_devfreq *tegra,
493					  struct tegra_devfreq_device *dev)
494{
495	u32 val = 0;
496
497	/* reset boosting on governor's restart */
498	dev->boost_freq = 0;
499
500	dev->target_freq = tegra->cur_freq;
501
502	dev->avg_count = tegra->cur_freq * tegra->devfreq->profile->polling_ms;
503	device_writel(dev, dev->avg_count, ACTMON_DEV_INIT_AVG);
504
505	tegra_devfreq_update_avg_wmark(tegra, dev);
506	tegra_devfreq_update_wmark(tegra, dev);
507
508	device_writel(dev, tegra->soc->count_weight, ACTMON_DEV_COUNT_WEIGHT);
509	device_writel(dev, ACTMON_INTR_STATUS_CLEAR, ACTMON_DEV_INTR_STATUS);
510
511	val |= ACTMON_DEV_CTRL_ENB_PERIODIC;
512	val |= (ACTMON_AVERAGE_WINDOW_LOG2 - 1)
513		<< ACTMON_DEV_CTRL_K_VAL_SHIFT;
514	val |= (ACTMON_BELOW_WMARK_WINDOW - 1)
515		<< ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_NUM_SHIFT;
516	val |= (ACTMON_ABOVE_WMARK_WINDOW - 1)
517		<< ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_NUM_SHIFT;
518	val |= ACTMON_DEV_CTRL_AVG_ABOVE_WMARK_EN;
519	val |= ACTMON_DEV_CTRL_AVG_BELOW_WMARK_EN;
520	val |= ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN;
521	val |= ACTMON_DEV_CTRL_ENB;
522
523	device_writel(dev, val, ACTMON_DEV_CTRL);
524}
525
526static void tegra_actmon_stop_devices(struct tegra_devfreq *tegra)
527{
528	struct tegra_devfreq_device *dev = tegra->devices;
529	unsigned int i;
530
531	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++, dev++) {
532		device_writel(dev, ACTMON_DEV_CTRL_STOP, ACTMON_DEV_CTRL);
533		device_writel(dev, ACTMON_INTR_STATUS_CLEAR,
534			      ACTMON_DEV_INTR_STATUS);
535	}
536}
537
538static int tegra_actmon_resume(struct tegra_devfreq *tegra)
539{
540	unsigned int i;
541	int err;
542
543	if (!tegra->devfreq->profile->polling_ms || !tegra->started)
544		return 0;
545
546	actmon_writel(tegra, tegra->devfreq->profile->polling_ms - 1,
547		      ACTMON_GLB_PERIOD_CTRL);
548
549	/*
550	 * CLK notifications are needed in order to reconfigure the upper
551	 * consecutive watermark in accordance to the actual clock rate
552	 * to avoid unnecessary upper interrupts.
553	 */
554	err = clk_notifier_register(tegra->emc_clock,
555				    &tegra->clk_rate_change_nb);
556	if (err) {
557		dev_err(tegra->devfreq->dev.parent,
558			"Failed to register rate change notifier\n");
559		return err;
560	}
561
562	tegra->cur_freq = clk_get_rate(tegra->emc_clock) / KHZ;
563
564	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++)
565		tegra_actmon_configure_device(tegra, &tegra->devices[i]);
566
567	/*
568	 * We are estimating CPU's memory bandwidth requirement based on
569	 * amount of memory accesses and system's load, judging by CPU's
570	 * frequency. We also don't want to receive events about CPU's
571	 * frequency transaction when governor is stopped, hence notifier
572	 * is registered dynamically.
573	 */
574	err = cpufreq_register_notifier(&tegra->cpu_rate_change_nb,
575					CPUFREQ_TRANSITION_NOTIFIER);
576	if (err) {
577		dev_err(tegra->devfreq->dev.parent,
578			"Failed to register rate change notifier: %d\n", err);
579		goto err_stop;
580	}
581
582	enable_irq(tegra->irq);
583
584	return 0;
585
586err_stop:
587	tegra_actmon_stop_devices(tegra);
588
589	clk_notifier_unregister(tegra->emc_clock, &tegra->clk_rate_change_nb);
590
591	return err;
592}
593
594static int tegra_actmon_start(struct tegra_devfreq *tegra)
595{
596	int ret = 0;
597
598	if (!tegra->started) {
599		tegra->started = true;
600
601		ret = tegra_actmon_resume(tegra);
602		if (ret)
603			tegra->started = false;
604	}
605
606	return ret;
607}
608
609static void tegra_actmon_pause(struct tegra_devfreq *tegra)
610{
611	if (!tegra->devfreq->profile->polling_ms || !tegra->started)
612		return;
613
614	disable_irq(tegra->irq);
615
616	cpufreq_unregister_notifier(&tegra->cpu_rate_change_nb,
617				    CPUFREQ_TRANSITION_NOTIFIER);
618
619	cancel_delayed_work_sync(&tegra->cpufreq_update_work);
620
621	tegra_actmon_stop_devices(tegra);
622
623	clk_notifier_unregister(tegra->emc_clock, &tegra->clk_rate_change_nb);
624}
625
626static void tegra_actmon_stop(struct tegra_devfreq *tegra)
627{
628	tegra_actmon_pause(tegra);
629	tegra->started = false;
630}
631
632static int tegra_devfreq_target(struct device *dev, unsigned long *freq,
633				u32 flags)
634{
635	struct dev_pm_opp *opp;
636	int ret;
637
638	opp = devfreq_recommended_opp(dev, freq, flags);
639	if (IS_ERR(opp)) {
640		dev_err(dev, "Failed to find opp for %lu Hz\n", *freq);
641		return PTR_ERR(opp);
642	}
643
644	ret = dev_pm_opp_set_opp(dev, opp);
645	dev_pm_opp_put(opp);
646
647	return ret;
648}
649
650static int tegra_devfreq_get_dev_status(struct device *dev,
651					struct devfreq_dev_status *stat)
652{
653	struct tegra_devfreq *tegra = dev_get_drvdata(dev);
654	struct tegra_devfreq_device *actmon_dev;
655	unsigned long cur_freq;
656
657	cur_freq = READ_ONCE(tegra->cur_freq);
658
659	/* To be used by the tegra governor */
660	stat->private_data = tegra;
661
662	/* The below are to be used by the other governors */
663	stat->current_frequency = cur_freq * KHZ;
664
665	actmon_dev = &tegra->devices[MCALL];
666
667	/* Number of cycles spent on memory access */
668	stat->busy_time = device_readl(actmon_dev, ACTMON_DEV_AVG_COUNT);
669
670	/* The bus can be considered to be saturated way before 100% */
671	stat->busy_time *= 100 / BUS_SATURATION_RATIO;
672
673	/* Number of cycles in a sampling period */
674	stat->total_time = tegra->devfreq->profile->polling_ms * cur_freq;
675
676	stat->busy_time = min(stat->busy_time, stat->total_time);
677
678	return 0;
679}
680
681static struct devfreq_dev_profile tegra_devfreq_profile = {
682	.polling_ms	= ACTMON_SAMPLING_PERIOD,
683	.target		= tegra_devfreq_target,
684	.get_dev_status	= tegra_devfreq_get_dev_status,
685	.is_cooling_device = true,
686};
687
688static int tegra_governor_get_target(struct devfreq *devfreq,
689				     unsigned long *freq)
690{
691	struct devfreq_dev_status *stat;
692	struct tegra_devfreq *tegra;
693	struct tegra_devfreq_device *dev;
694	unsigned long target_freq = 0;
695	unsigned int i;
696	int err;
697
698	err = devfreq_update_stats(devfreq);
699	if (err)
700		return err;
701
702	stat = &devfreq->last_status;
703
704	tegra = stat->private_data;
705
706	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
707		dev = &tegra->devices[i];
708
709		actmon_update_target(tegra, dev);
710
711		target_freq = max(target_freq, dev->target_freq);
712	}
713
714	/*
715	 * tegra-devfreq driver operates with KHz units, while OPP table
716	 * entries use Hz units. Hence we need to convert the units for the
717	 * devfreq core.
718	 */
719	*freq = target_freq * KHZ;
720
721	return 0;
722}
723
724static int tegra_governor_event_handler(struct devfreq *devfreq,
725					unsigned int event, void *data)
726{
727	struct tegra_devfreq *tegra = dev_get_drvdata(devfreq->dev.parent);
728	unsigned int *new_delay = data;
729	int ret = 0;
730
731	/*
732	 * Couple devfreq-device with the governor early because it is
733	 * needed at the moment of governor's start (used by ISR).
734	 */
735	tegra->devfreq = devfreq;
736
737	switch (event) {
738	case DEVFREQ_GOV_START:
739		devfreq_monitor_start(devfreq);
740		ret = tegra_actmon_start(tegra);
741		break;
742
743	case DEVFREQ_GOV_STOP:
744		tegra_actmon_stop(tegra);
745		devfreq_monitor_stop(devfreq);
746		break;
747
748	case DEVFREQ_GOV_UPDATE_INTERVAL:
749		/*
750		 * ACTMON hardware supports up to 256 milliseconds for the
751		 * sampling period.
752		 */
753		if (*new_delay > 256) {
754			ret = -EINVAL;
755			break;
756		}
757
758		tegra_actmon_pause(tegra);
759		devfreq_update_interval(devfreq, new_delay);
760		ret = tegra_actmon_resume(tegra);
761		break;
762
763	case DEVFREQ_GOV_SUSPEND:
764		tegra_actmon_stop(tegra);
765		devfreq_monitor_suspend(devfreq);
766		break;
767
768	case DEVFREQ_GOV_RESUME:
769		devfreq_monitor_resume(devfreq);
770		ret = tegra_actmon_start(tegra);
771		break;
772	}
773
774	return ret;
775}
776
777static struct devfreq_governor tegra_devfreq_governor = {
778	.name = "tegra_actmon",
779	.attrs = DEVFREQ_GOV_ATTR_POLLING_INTERVAL,
780	.flags = DEVFREQ_GOV_FLAG_IMMUTABLE
781		| DEVFREQ_GOV_FLAG_IRQ_DRIVEN,
782	.get_target_freq = tegra_governor_get_target,
783	.event_handler = tegra_governor_event_handler,
784};
785
786static void devm_tegra_devfreq_deinit_hw(void *data)
787{
788	struct tegra_devfreq *tegra = data;
789
790	reset_control_reset(tegra->reset);
791	clk_disable_unprepare(tegra->clock);
792}
793
794static int devm_tegra_devfreq_init_hw(struct device *dev,
795				      struct tegra_devfreq *tegra)
796{
797	int err;
798
799	err = clk_prepare_enable(tegra->clock);
800	if (err) {
801		dev_err(dev, "Failed to prepare and enable ACTMON clock\n");
802		return err;
803	}
804
805	err = devm_add_action_or_reset(dev, devm_tegra_devfreq_deinit_hw,
806				       tegra);
807	if (err)
808		return err;
809
810	err = reset_control_reset(tegra->reset);
811	if (err) {
812		dev_err(dev, "Failed to reset hardware: %d\n", err);
813		return err;
814	}
815
816	return err;
817}
818
819static int tegra_devfreq_config_clks_nop(struct device *dev,
820					 struct opp_table *opp_table,
821					 struct dev_pm_opp *opp, void *data,
822					 bool scaling_down)
823{
824	/* We want to skip clk configuration via dev_pm_opp_set_opp() */
825	return 0;
826}
827
828static int tegra_devfreq_probe(struct platform_device *pdev)
829{
830	u32 hw_version = BIT(tegra_sku_info.soc_speedo_id);
831	struct tegra_devfreq_device *dev;
832	struct tegra_devfreq *tegra;
833	struct devfreq *devfreq;
834	unsigned int i;
835	long rate;
836	int err;
837	const char *clk_names[] = { "actmon", NULL };
838	struct dev_pm_opp_config config = {
839		.supported_hw = &hw_version,
840		.supported_hw_count = 1,
841		.clk_names = clk_names,
842		.config_clks = tegra_devfreq_config_clks_nop,
843	};
844
845	tegra = devm_kzalloc(&pdev->dev, sizeof(*tegra), GFP_KERNEL);
846	if (!tegra)
847		return -ENOMEM;
848
849	tegra->soc = of_device_get_match_data(&pdev->dev);
850
851	tegra->regs = devm_platform_ioremap_resource(pdev, 0);
852	if (IS_ERR(tegra->regs))
853		return PTR_ERR(tegra->regs);
854
855	tegra->reset = devm_reset_control_get(&pdev->dev, "actmon");
856	if (IS_ERR(tegra->reset)) {
857		dev_err(&pdev->dev, "Failed to get reset\n");
858		return PTR_ERR(tegra->reset);
859	}
860
861	tegra->clock = devm_clk_get(&pdev->dev, "actmon");
862	if (IS_ERR(tegra->clock)) {
863		dev_err(&pdev->dev, "Failed to get actmon clock\n");
864		return PTR_ERR(tegra->clock);
865	}
866
867	tegra->emc_clock = devm_clk_get(&pdev->dev, "emc");
868	if (IS_ERR(tegra->emc_clock))
869		return dev_err_probe(&pdev->dev, PTR_ERR(tegra->emc_clock),
870				     "Failed to get emc clock\n");
871
872	err = platform_get_irq(pdev, 0);
873	if (err < 0)
874		return err;
875
876	tegra->irq = err;
877
878	irq_set_status_flags(tegra->irq, IRQ_NOAUTOEN);
879
880	err = devm_request_threaded_irq(&pdev->dev, tegra->irq, NULL,
881					actmon_thread_isr, IRQF_ONESHOT,
882					"tegra-devfreq", tegra);
883	if (err) {
884		dev_err(&pdev->dev, "Interrupt request failed: %d\n", err);
885		return err;
886	}
887
888	err = devm_pm_opp_set_config(&pdev->dev, &config);
889	if (err) {
890		dev_err(&pdev->dev, "Failed to set OPP config: %d\n", err);
891		return err;
892	}
893
894	err = devm_pm_opp_of_add_table_indexed(&pdev->dev, 0);
895	if (err) {
896		dev_err(&pdev->dev, "Failed to add OPP table: %d\n", err);
897		return err;
898	}
899
900	err = devm_tegra_devfreq_init_hw(&pdev->dev, tegra);
901	if (err)
902		return err;
903
904	rate = clk_round_rate(tegra->emc_clock, ULONG_MAX);
905	if (rate <= 0) {
906		dev_err(&pdev->dev, "Failed to round clock rate: %ld\n", rate);
907		return rate ?: -EINVAL;
908	}
909
910	tegra->max_freq = rate / KHZ;
911
912	for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) {
913		dev = tegra->devices + i;
914		dev->config = tegra->soc->configs + i;
915		dev->regs = tegra->regs + dev->config->offset;
916	}
917
918	platform_set_drvdata(pdev, tegra);
919
920	tegra->clk_rate_change_nb.notifier_call = tegra_actmon_clk_notify_cb;
921	tegra->cpu_rate_change_nb.notifier_call = tegra_actmon_cpu_notify_cb;
922
923	INIT_DELAYED_WORK(&tegra->cpufreq_update_work,
924			  tegra_actmon_delayed_update);
925
926	err = devm_devfreq_add_governor(&pdev->dev, &tegra_devfreq_governor);
927	if (err) {
928		dev_err(&pdev->dev, "Failed to add governor: %d\n", err);
929		return err;
930	}
931
932	tegra_devfreq_profile.initial_freq = clk_get_rate(tegra->emc_clock);
933
934	devfreq = devm_devfreq_add_device(&pdev->dev, &tegra_devfreq_profile,
935					  "tegra_actmon", NULL);
936	if (IS_ERR(devfreq)) {
937		dev_err(&pdev->dev, "Failed to add device: %pe\n", devfreq);
938		return PTR_ERR(devfreq);
939	}
940
941	return 0;
942}
943
944static const struct tegra_devfreq_soc_data tegra124_soc = {
945	.configs = tegra124_device_configs,
946
947	/*
948	 * Activity counter is incremented every 256 memory transactions,
949	 * and each transaction takes 4 EMC clocks.
950	 */
951	.count_weight = 4 * 256,
952};
953
954static const struct tegra_devfreq_soc_data tegra30_soc = {
955	.configs = tegra30_device_configs,
956	.count_weight = 2 * 256,
957};
958
959static const struct of_device_id tegra_devfreq_of_match[] = {
960	{ .compatible = "nvidia,tegra30-actmon",  .data = &tegra30_soc, },
961	{ .compatible = "nvidia,tegra124-actmon", .data = &tegra124_soc, },
962	{ },
963};
964
965MODULE_DEVICE_TABLE(of, tegra_devfreq_of_match);
966
967static struct platform_driver tegra_devfreq_driver = {
968	.probe	= tegra_devfreq_probe,
969	.driver = {
970		.name = "tegra-devfreq",
971		.of_match_table = tegra_devfreq_of_match,
972	},
973};
974module_platform_driver(tegra_devfreq_driver);
975
976MODULE_LICENSE("GPL v2");
977MODULE_DESCRIPTION("Tegra devfreq driver");
978MODULE_AUTHOR("Tomeu Vizoso <tomeu.vizoso@collabora.com>");