apple-soc-cpufreq.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Apple SoC CPU cluster performance state driver
  4 *
  5 * Copyright The Asahi Linux Contributors
  6 *
  7 * Based on scpi-cpufreq.c
  8 */
  9
 10#include <linux/bitfield.h>
 11#include <linux/bitops.h>
 12#include <linux/cpu.h>
 13#include <linux/cpufreq.h>
 14#include <linux/cpumask.h>
 15#include <linux/delay.h>
 16#include <linux/err.h>
 17#include <linux/io.h>
 18#include <linux/iopoll.h>
 19#include <linux/module.h>
 20#include <linux/of.h>
 21#include <linux/of_address.h>
 22#include <linux/pm_opp.h>
 23#include <linux/slab.h>
 24
 25#define APPLE_DVFS_CMD				0x20
 26#define APPLE_DVFS_CMD_BUSY			BIT(31)
 27#define APPLE_DVFS_CMD_SET			BIT(25)
 28#define APPLE_DVFS_CMD_PS1_S5L8960X		GENMASK(24, 22)
 29#define APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT	22
 30#define APPLE_DVFS_CMD_PS2			GENMASK(15, 12)
 31#define APPLE_DVFS_CMD_PS1			GENMASK(4, 0)
 32#define APPLE_DVFS_CMD_PS1_SHIFT		0
 33
 34/* Same timebase as CPU counter (24MHz) */
 35#define APPLE_DVFS_LAST_CHG_TIME	0x38
 36
 37/*
 38 * Apple ran out of bits and had to shift this in T8112...
 39 */
 40#define APPLE_DVFS_STATUS			0x50
 41#define APPLE_DVFS_STATUS_CUR_PS_S5L8960X	GENMASK(5, 3)
 42#define APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X	3
 43#define APPLE_DVFS_STATUS_TGT_PS_S5L8960X	GENMASK(2, 0)
 44#define APPLE_DVFS_STATUS_CUR_PS_T8103		GENMASK(7, 4)
 45#define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103	4
 46#define APPLE_DVFS_STATUS_TGT_PS_T8103		GENMASK(3, 0)
 47#define APPLE_DVFS_STATUS_CUR_PS_T8112		GENMASK(9, 5)
 48#define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112	5
 49#define APPLE_DVFS_STATUS_TGT_PS_T8112		GENMASK(4, 0)
 50
 51/*
 52 * Div is +1, base clock is 12MHz on existing SoCs.
 53 * For documentation purposes. We use the OPP table to
 54 * get the frequency.
 55 */
 56#define APPLE_DVFS_PLL_STATUS		0xc0
 57#define APPLE_DVFS_PLL_FACTOR		0xc8
 58#define APPLE_DVFS_PLL_FACTOR_MULT	GENMASK(31, 16)
 59#define APPLE_DVFS_PLL_FACTOR_DIV	GENMASK(15, 0)
 60
 61#define APPLE_DVFS_TRANSITION_TIMEOUT 400
 62
 63struct apple_soc_cpufreq_info {
 64	bool has_ps2;
 65	u64 max_pstate;
 66	u64 cur_pstate_mask;
 67	u64 cur_pstate_shift;
 68	u64 ps1_mask;
 69	u64 ps1_shift;
 70};
 71
 72struct apple_cpu_priv {
 73	struct device *cpu_dev;
 74	void __iomem *reg_base;
 75	const struct apple_soc_cpufreq_info *info;
 76};
 77
 78static struct cpufreq_driver apple_soc_cpufreq_driver;
 79
 80static const struct apple_soc_cpufreq_info soc_s5l8960x_info = {
 81	.has_ps2 = false,
 82	.max_pstate = 7,
 83	.cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_S5L8960X,
 84	.cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_S5L8960X,
 85	.ps1_mask = APPLE_DVFS_CMD_PS1_S5L8960X,
 86	.ps1_shift = APPLE_DVFS_CMD_PS1_S5L8960X_SHIFT,
 87};
 88
 89static const struct apple_soc_cpufreq_info soc_t8103_info = {
 90	.has_ps2 = true,
 91	.max_pstate = 15,
 92	.cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103,
 93	.cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103,
 94	.ps1_mask = APPLE_DVFS_CMD_PS1,
 95	.ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT,
 96};
 97
 98static const struct apple_soc_cpufreq_info soc_t8112_info = {
 99	.has_ps2 = false,
100	.max_pstate = 31,
101	.cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112,
102	.cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112,
103	.ps1_mask = APPLE_DVFS_CMD_PS1,
104	.ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT,
105};
106
107static const struct apple_soc_cpufreq_info soc_default_info = {
108	.has_ps2 = false,
109	.max_pstate = 15,
110	.cur_pstate_mask = 0, /* fallback */
111	.ps1_mask = APPLE_DVFS_CMD_PS1,
112	.ps1_shift = APPLE_DVFS_CMD_PS1_SHIFT,
113};
114
115static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = {
116	{
117		.compatible = "apple,s5l8960x-cluster-cpufreq",
118		.data = &soc_s5l8960x_info,
119	},
120	{
121		.compatible = "apple,t8103-cluster-cpufreq",
122		.data = &soc_t8103_info,
123	},
124	{
125		.compatible = "apple,t8112-cluster-cpufreq",
126		.data = &soc_t8112_info,
127	},
128	{
129		.compatible = "apple,cluster-cpufreq",
130		.data = &soc_default_info,
131	},
132	{}
133};
134
135static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu)
136{
137	struct cpufreq_policy *policy;
138	struct apple_cpu_priv *priv;
139	struct cpufreq_frequency_table *p;
140	unsigned int pstate;
141
142	policy = cpufreq_cpu_get_raw(cpu);
143	if (unlikely(!policy))
144		return 0;
145
146	priv = policy->driver_data;
147
148	if (priv->info->cur_pstate_mask) {
149		u32 reg = readl_relaxed(priv->reg_base + APPLE_DVFS_STATUS);
150
151		pstate = (reg & priv->info->cur_pstate_mask) >>  priv->info->cur_pstate_shift;
152	} else {
153		/*
154		 * For the fallback case we might not know the layout of DVFS_STATUS,
155		 * so just use the command register value (which ignores boost limitations).
156		 */
157		u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD);
158
159		pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg);
160	}
161
162	cpufreq_for_each_valid_entry(p, policy->freq_table)
163		if (p->driver_data == pstate)
164			return p->frequency;
165
166	dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n",
167		pstate);
168	return 0;
169}
170
171static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy,
172					unsigned int index)
173{
174	struct apple_cpu_priv *priv = policy->driver_data;
175	unsigned int pstate = policy->freq_table[index].driver_data;
176	u64 reg;
177
178	/* Fallback for newer SoCs */
179	if (index > priv->info->max_pstate)
180		index = priv->info->max_pstate;
181
182	if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg,
183				      !(reg & APPLE_DVFS_CMD_BUSY), 2,
184				      APPLE_DVFS_TRANSITION_TIMEOUT)) {
185		return -EIO;
186	}
187
188	reg &= ~priv->info->ps1_mask;
189	reg |= pstate << priv->info->ps1_shift;
190	if (priv->info->has_ps2) {
191		reg &= ~APPLE_DVFS_CMD_PS2;
192		reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate);
193	}
194	reg |= APPLE_DVFS_CMD_SET;
195
196	writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD);
197
198	return 0;
199}
200
201static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy,
202						  unsigned int target_freq)
203{
204	if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0)
205		return 0;
206
207	return policy->freq_table[policy->cached_resolved_idx].frequency;
208}
209
210static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy,
211					  void __iomem **reg_base,
212					  const struct apple_soc_cpufreq_info **info)
213{
214	struct of_phandle_args args;
215	const struct of_device_id *match;
216	int ret = 0;
217
218	ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains",
219						 "#performance-domain-cells",
220						 policy->cpus, &args);
221	if (ret < 0)
222		return ret;
223
224	match = of_match_node(apple_soc_cpufreq_of_match, args.np);
225	of_node_put(args.np);
226	if (!match)
227		return -ENODEV;
228
229	*info = match->data;
230
231	*reg_base = of_iomap(args.np, 0);
232	if (!*reg_base)
233		return -ENOMEM;
234
235	return 0;
236}
237
238static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
239{
240	int ret, i;
241	unsigned int transition_latency;
242	void __iomem *reg_base;
243	struct device *cpu_dev;
244	struct apple_cpu_priv *priv;
245	const struct apple_soc_cpufreq_info *info;
246	struct cpufreq_frequency_table *freq_table;
247
248	cpu_dev = get_cpu_device(policy->cpu);
249	if (!cpu_dev) {
250		pr_err("failed to get cpu%d device\n", policy->cpu);
251		return -ENODEV;
252	}
253
254	ret = dev_pm_opp_of_add_table(cpu_dev);
255	if (ret < 0) {
256		dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);
257		return ret;
258	}
259
260	ret = apple_soc_cpufreq_find_cluster(policy, &reg_base, &info);
261	if (ret) {
262		dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret);
263		return ret;
264	}
265
266	ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
267	if (ret) {
268		dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret);
269		goto out_iounmap;
270	}
271
272	ret = dev_pm_opp_get_opp_count(cpu_dev);
273	if (ret <= 0) {
274		dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
275		ret = -EPROBE_DEFER;
276		goto out_free_opp;
277	}
278
279	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
280	if (!priv) {
281		ret = -ENOMEM;
282		goto out_free_opp;
283	}
284
285	ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
286	if (ret) {
287		dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
288		goto out_free_priv;
289	}
290
291	/* Get OPP levels (p-state indexes) and stash them in driver_data */
292	for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
293		unsigned long rate = freq_table[i].frequency * 1000 + 999;
294		struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate);
295
296		if (IS_ERR(opp)) {
297			ret = PTR_ERR(opp);
298			goto out_free_cpufreq_table;
299		}
300		freq_table[i].driver_data = dev_pm_opp_get_level(opp);
301		dev_pm_opp_put(opp);
302	}
303
304	priv->cpu_dev = cpu_dev;
305	priv->reg_base = reg_base;
306	priv->info = info;
307	policy->driver_data = priv;
308	policy->freq_table = freq_table;
309
310	transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev);
311	if (!transition_latency)
312		transition_latency = APPLE_DVFS_TRANSITION_TIMEOUT * NSEC_PER_USEC;
313
314	policy->cpuinfo.transition_latency = transition_latency;
315	policy->dvfs_possible_from_any_cpu = true;
316	policy->fast_switch_possible = true;
317	policy->suspend_freq = freq_table[0].frequency;
318
319	return 0;
320
321out_free_cpufreq_table:
322	dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
323out_free_priv:
324	kfree(priv);
325out_free_opp:
326	dev_pm_opp_remove_all_dynamic(cpu_dev);
327out_iounmap:
328	iounmap(reg_base);
329	return ret;
330}
331
332static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy)
333{
334	struct apple_cpu_priv *priv = policy->driver_data;
335
336	dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
337	dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
338	iounmap(priv->reg_base);
339	kfree(priv);
340}
341
342static struct cpufreq_driver apple_soc_cpufreq_driver = {
343	.name		= "apple-cpufreq",
344	.flags		= CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
345			  CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV,
346	.verify		= cpufreq_generic_frequency_table_verify,
347	.get		= apple_soc_cpufreq_get_rate,
348	.init		= apple_soc_cpufreq_init,
349	.exit		= apple_soc_cpufreq_exit,
350	.target_index	= apple_soc_cpufreq_set_target,
351	.fast_switch	= apple_soc_cpufreq_fast_switch,
352	.register_em	= cpufreq_register_em_with_opp,
353	.set_boost	= cpufreq_boost_set_sw,
354	.suspend	= cpufreq_generic_suspend,
355};
356
357static int __init apple_soc_cpufreq_module_init(void)
358{
359	if (!of_machine_is_compatible("apple,arm-platform"))
360		return -ENODEV;
361
362	return cpufreq_register_driver(&apple_soc_cpufreq_driver);
363}
364module_init(apple_soc_cpufreq_module_init);
365
366static void __exit apple_soc_cpufreq_module_exit(void)
367{
368	cpufreq_unregister_driver(&apple_soc_cpufreq_driver);
369}
370module_exit(apple_soc_cpufreq_module_exit);
371
372MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match);
373MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");
374MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver");
375MODULE_LICENSE("GPL");