hisi_uncore_freq.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * HiSilicon uncore frequency scaling driver
  4 *
  5 * Copyright (c) 2025 HiSilicon Co., Ltd
  6 */
  7
  8#include <linux/acpi.h>
  9#include <linux/bits.h>
 10#include <linux/cleanup.h>
 11#include <linux/devfreq.h>
 12#include <linux/devfreq-governor.h>
 13#include <linux/device.h>
 14#include <linux/dev_printk.h>
 15#include <linux/errno.h>
 16#include <linux/iopoll.h>
 17#include <linux/kernel.h>
 18#include <linux/ktime.h>
 19#include <linux/mailbox_client.h>
 20#include <linux/module.h>
 21#include <linux/mod_devicetable.h>
 22#include <linux/mutex.h>
 23#include <linux/platform_device.h>
 24#include <linux/pm_opp.h>
 25#include <linux/property.h>
 26#include <linux/topology.h>
 27#include <linux/units.h>
 28#include <acpi/pcc.h>
 29
 30struct hisi_uncore_pcc_data {
 31	u16 status;
 32	u16 resv;
 33	u32 data;
 34};
 35
 36struct hisi_uncore_pcc_shmem {
 37	struct acpi_pcct_shared_memory head;
 38	struct hisi_uncore_pcc_data pcc_data;
 39};
 40
 41enum hisi_uncore_pcc_cmd_type {
 42	HUCF_PCC_CMD_GET_CAP = 0,
 43	HUCF_PCC_CMD_GET_FREQ,
 44	HUCF_PCC_CMD_SET_FREQ,
 45	HUCF_PCC_CMD_GET_MODE,
 46	HUCF_PCC_CMD_SET_MODE,
 47	HUCF_PCC_CMD_GET_PLAT_FREQ_NUM,
 48	HUCF_PCC_CMD_GET_PLAT_FREQ_BY_IDX,
 49	HUCF_PCC_CMD_MAX = 256
 50};
 51
 52static int hisi_platform_gov_usage;
 53static DEFINE_MUTEX(hisi_platform_gov_usage_lock);
 54
 55enum hisi_uncore_freq_mode {
 56	HUCF_MODE_PLATFORM = 0,
 57	HUCF_MODE_OS,
 58	HUCF_MODE_MAX
 59};
 60
 61#define HUCF_CAP_PLATFORM_CTRL	BIT(0)
 62
 63/**
 64 * struct hisi_uncore_freq - hisi uncore frequency scaling device data
 65 * @dev:		device of this frequency scaling driver
 66 * @cl:			mailbox client object
 67 * @pchan:		PCC mailbox channel
 68 * @chan_id:		PCC channel ID
 69 * @last_cmd_cmpl_time:	timestamp of the last completed PCC command
 70 * @pcc_lock:		PCC channel lock
 71 * @devfreq:		devfreq data of this hisi_uncore_freq device
 72 * @related_cpus:	CPUs whose performance is majorly affected by this
 73 *			uncore frequency domain
 74 * @cap:		capability flag
 75 */
 76struct hisi_uncore_freq {
 77	struct device *dev;
 78	struct mbox_client cl;
 79	struct pcc_mbox_chan *pchan;
 80	int chan_id;
 81	ktime_t last_cmd_cmpl_time;
 82	struct mutex pcc_lock;
 83	struct devfreq *devfreq;
 84	struct cpumask related_cpus;
 85	u32 cap;
 86};
 87
 88/* PCC channel timeout = PCC nominal latency * NUM */
 89#define HUCF_PCC_POLL_TIMEOUT_NUM	1000
 90#define HUCF_PCC_POLL_INTERVAL_US	5
 91
 92/* Default polling interval in ms for devfreq governors*/
 93#define HUCF_DEFAULT_POLLING_MS 100
 94
 95static void hisi_uncore_free_pcc_chan(struct hisi_uncore_freq *uncore)
 96{
 97	guard(mutex)(&uncore->pcc_lock);
 98	pcc_mbox_free_channel(uncore->pchan);
 99	uncore->pchan = NULL;
100}
101
102static void devm_hisi_uncore_free_pcc_chan(void *data)
103{
104	hisi_uncore_free_pcc_chan(data);
105}
106
107static int hisi_uncore_request_pcc_chan(struct hisi_uncore_freq *uncore)
108{
109	struct device *dev = uncore->dev;
110	struct pcc_mbox_chan *pcc_chan;
111
112	uncore->cl = (struct mbox_client) {
113		.dev = dev,
114		.tx_block = false,
115		.knows_txdone = true,
116	};
117
118	pcc_chan = pcc_mbox_request_channel(&uncore->cl, uncore->chan_id);
119	if (IS_ERR(pcc_chan))
120		return dev_err_probe(dev, PTR_ERR(pcc_chan),
121			"Failed to request PCC channel %u\n", uncore->chan_id);
122
123	if (!pcc_chan->shmem_base_addr) {
124		pcc_mbox_free_channel(pcc_chan);
125		return dev_err_probe(dev, -EINVAL,
126			"Invalid PCC shared memory address\n");
127	}
128
129	if (pcc_chan->shmem_size < sizeof(struct hisi_uncore_pcc_shmem)) {
130		pcc_mbox_free_channel(pcc_chan);
131		return dev_err_probe(dev, -EINVAL,
132			"Invalid PCC shared memory size (%lluB)\n",
133			pcc_chan->shmem_size);
134	}
135
136	uncore->pchan = pcc_chan;
137
138	return devm_add_action_or_reset(uncore->dev,
139					devm_hisi_uncore_free_pcc_chan, uncore);
140}
141
142static acpi_status hisi_uncore_pcc_reg_scan(struct acpi_resource *res,
143					    void *ctx)
144{
145	struct acpi_resource_generic_register *reg;
146	struct hisi_uncore_freq *uncore;
147
148	if (!res || res->type != ACPI_RESOURCE_TYPE_GENERIC_REGISTER)
149		return AE_OK;
150
151	reg = &res->data.generic_reg;
152	if (reg->space_id != ACPI_ADR_SPACE_PLATFORM_COMM)
153		return AE_OK;
154
155	if (!ctx)
156		return AE_ERROR;
157
158	uncore = ctx;
159	/* PCC subspace ID stored in Access Size */
160	uncore->chan_id = reg->access_size;
161
162	return AE_CTRL_TERMINATE;
163}
164
165static int hisi_uncore_init_pcc_chan(struct hisi_uncore_freq *uncore)
166{
167	acpi_handle handle = ACPI_HANDLE(uncore->dev);
168	acpi_status status;
169	int rc;
170
171	uncore->chan_id = -1;
172	status = acpi_walk_resources(handle, METHOD_NAME__CRS,
173				     hisi_uncore_pcc_reg_scan, uncore);
174	if (ACPI_FAILURE(status) || uncore->chan_id < 0)
175		return dev_err_probe(uncore->dev, -ENODEV,
176			"Failed to get a PCC channel\n");
177
178
179	rc = devm_mutex_init(uncore->dev, &uncore->pcc_lock);
180	if (rc)
181		return rc;
182
183	return hisi_uncore_request_pcc_chan(uncore);
184}
185
186static int hisi_uncore_cmd_send(struct hisi_uncore_freq *uncore,
187				u8 cmd, u32 *data)
188{
189	struct hisi_uncore_pcc_shmem __iomem *addr;
190	struct hisi_uncore_pcc_shmem shmem;
191	struct pcc_mbox_chan *pchan;
192	unsigned int mrtt;
193	s64 time_delta;
194	u16 status;
195	int rc;
196
197	guard(mutex)(&uncore->pcc_lock);
198
199	pchan = uncore->pchan;
200	if (!pchan)
201		return -ENODEV;
202
203	addr = (struct hisi_uncore_pcc_shmem __iomem *)pchan->shmem;
204	if (!addr)
205		return -EINVAL;
206
207	/* Handle the Minimum Request Turnaround Time (MRTT) */
208	mrtt = pchan->min_turnaround_time;
209	time_delta = ktime_us_delta(ktime_get(), uncore->last_cmd_cmpl_time);
210	if (mrtt > time_delta)
211		udelay(mrtt - time_delta);
212
213	/* Copy data */
214	shmem.head = (struct acpi_pcct_shared_memory) {
215		.signature = PCC_SIGNATURE | uncore->chan_id,
216		.command = cmd,
217	};
218	shmem.pcc_data.data = *data;
219	memcpy_toio(addr, &shmem, sizeof(shmem));
220
221	/* Ring doorbell */
222	rc = mbox_send_message(pchan->mchan, &cmd);
223	if (rc < 0) {
224		dev_err(uncore->dev, "Failed to send mbox message, %d\n", rc);
225		return rc;
226	}
227
228	/* Wait status */
229	rc = readw_poll_timeout(&addr->head.status, status,
230				status & (PCC_STATUS_CMD_COMPLETE |
231					  PCC_STATUS_ERROR),
232				HUCF_PCC_POLL_INTERVAL_US,
233				pchan->latency * HUCF_PCC_POLL_TIMEOUT_NUM);
234	if (rc) {
235		dev_err(uncore->dev, "PCC channel response timeout, cmd=%u\n", cmd);
236	} else if (status & PCC_STATUS_ERROR) {
237		dev_err(uncore->dev, "PCC cmd error, cmd=%u\n", cmd);
238		rc = -EIO;
239	}
240
241	uncore->last_cmd_cmpl_time = ktime_get();
242
243	/* Copy data back */
244	memcpy_fromio(data, &addr->pcc_data.data, sizeof(*data));
245
246	/* Clear mailbox active req */
247	mbox_client_txdone(pchan->mchan, rc);
248
249	return rc;
250}
251
252static int hisi_uncore_target(struct device *dev, unsigned long *freq,
253			      u32 flags)
254{
255	struct hisi_uncore_freq *uncore = dev_get_drvdata(dev);
256	struct dev_pm_opp *opp;
257	u32 data;
258
259	if (WARN_ON(!uncore || !uncore->pchan))
260		return -ENODEV;
261
262	opp = devfreq_recommended_opp(dev, freq, flags);
263	if (IS_ERR(opp)) {
264		dev_err(dev, "Failed to get opp for freq %lu hz\n", *freq);
265		return PTR_ERR(opp);
266	}
267
268	data = (u32)(dev_pm_opp_get_freq(opp) / HZ_PER_MHZ);
269
270	dev_pm_opp_put(opp);
271
272	return hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_FREQ, &data);
273}
274
275static int hisi_uncore_get_dev_status(struct device *dev,
276				      struct devfreq_dev_status *stat)
277{
278	/* Not used */
279	return 0;
280}
281
282static int hisi_uncore_get_cur_freq(struct device *dev, unsigned long *freq)
283{
284	struct hisi_uncore_freq *uncore = dev_get_drvdata(dev);
285	u32 data = 0;
286	int rc;
287
288	if (WARN_ON(!uncore || !uncore->pchan))
289		return -ENODEV;
290
291	rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_FREQ, &data);
292
293	/*
294	 * Upon a failure, 'data' remains 0 and 'freq' is set to 0 rather than a
295	 * random value.  devfreq shouldn't use 'freq' in that case though.
296	 */
297	*freq = data * HZ_PER_MHZ;
298
299	return rc;
300}
301
302static void devm_hisi_uncore_remove_opp(void *data)
303{
304	struct hisi_uncore_freq *uncore = data;
305
306	dev_pm_opp_remove_all_dynamic(uncore->dev);
307}
308
309static int hisi_uncore_init_opp(struct hisi_uncore_freq *uncore)
310{
311	struct device *dev = uncore->dev;
312	unsigned long freq_mhz;
313	u32 num, index;
314	u32 data = 0;
315	int rc;
316
317	rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_PLAT_FREQ_NUM,
318				  &data);
319	if (rc)
320		return dev_err_probe(dev, rc, "Failed to get plat freq num\n");
321
322	num = data;
323
324	for (index = 0; index < num; index++) {
325		data = index;
326		rc = hisi_uncore_cmd_send(uncore,
327					  HUCF_PCC_CMD_GET_PLAT_FREQ_BY_IDX,
328					  &data);
329		if (rc) {
330			dev_pm_opp_remove_all_dynamic(dev);
331			return dev_err_probe(dev, rc,
332				"Failed to get plat freq at index %u\n", index);
333		}
334		freq_mhz = data;
335
336		/* Don't care OPP voltage, take 1V as default */
337		rc = dev_pm_opp_add(dev, freq_mhz * HZ_PER_MHZ, 1000000);
338		if (rc) {
339			dev_pm_opp_remove_all_dynamic(dev);
340			return dev_err_probe(dev, rc,
341				"Add OPP %lu failed\n", freq_mhz);
342		}
343	}
344
345	return devm_add_action_or_reset(dev, devm_hisi_uncore_remove_opp,
346					uncore);
347}
348
349static int hisi_platform_gov_func(struct devfreq *df, unsigned long *freq)
350{
351	/*
352	 * Platform-controlled mode doesn't care the frequency issued from
353	 * devfreq, so just pick the max freq.
354	 */
355	*freq = DEVFREQ_MAX_FREQ;
356
357	return 0;
358}
359
360static int hisi_platform_gov_handler(struct devfreq *df, unsigned int event,
361				     void *val)
362{
363	struct hisi_uncore_freq *uncore = dev_get_drvdata(df->dev.parent);
364	int rc = 0;
365	u32 data;
366
367	if (WARN_ON(!uncore || !uncore->pchan))
368		return -ENODEV;
369
370	switch (event) {
371	case DEVFREQ_GOV_START:
372		data = HUCF_MODE_PLATFORM;
373		rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_MODE, &data);
374		if (rc)
375			dev_err(uncore->dev, "Failed to set platform mode (%d)\n", rc);
376		break;
377	case DEVFREQ_GOV_STOP:
378		data = HUCF_MODE_OS;
379		rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_MODE, &data);
380		if (rc)
381			dev_err(uncore->dev, "Failed to set os mode (%d)\n", rc);
382		break;
383	default:
384		break;
385	}
386
387	return rc;
388}
389
390/*
391 * In the platform-controlled mode, the platform decides the uncore frequency
392 * and ignores the frequency issued from the driver.
393 * Thus, create a pseudo 'hisi_platform' governor that stops devfreq monitor
394 * from working so as to save meaningless overhead.
395 */
396static struct devfreq_governor hisi_platform_governor = {
397	.name = "hisi_platform",
398	/*
399	 * Set interrupt_driven to skip the devfreq monitor mechanism, though
400	 * this governor is not interrupt-driven.
401	 */
402	.flags = DEVFREQ_GOV_FLAG_IRQ_DRIVEN,
403	.get_target_freq = hisi_platform_gov_func,
404	.event_handler = hisi_platform_gov_handler,
405};
406
407static void hisi_uncore_remove_platform_gov(struct hisi_uncore_freq *uncore)
408{
409	u32 data = HUCF_MODE_PLATFORM;
410	int rc;
411
412	if (!(uncore->cap & HUCF_CAP_PLATFORM_CTRL))
413		return;
414
415	guard(mutex)(&hisi_platform_gov_usage_lock);
416
417	if (--hisi_platform_gov_usage == 0) {
418		rc = devfreq_remove_governor(&hisi_platform_governor);
419		if (rc)
420			dev_err(uncore->dev, "Failed to remove hisi_platform gov (%d)\n", rc);
421	}
422
423	/*
424	 * Set to the platform-controlled mode on exit if supported, so as to
425	 * have a certain behaviour when the driver is detached.
426	 */
427	rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_MODE, &data);
428	if (rc)
429		dev_err(uncore->dev, "Failed to set platform mode on exit (%d)\n", rc);
430}
431
432static void devm_hisi_uncore_remove_platform_gov(void *data)
433{
434	hisi_uncore_remove_platform_gov(data);
435}
436
437static int hisi_uncore_add_platform_gov(struct hisi_uncore_freq *uncore)
438{
439	if (!(uncore->cap & HUCF_CAP_PLATFORM_CTRL))
440		return 0;
441
442	guard(mutex)(&hisi_platform_gov_usage_lock);
443
444	if (hisi_platform_gov_usage == 0) {
445		int rc = devfreq_add_governor(&hisi_platform_governor);
446		if (rc)
447			return rc;
448	}
449	hisi_platform_gov_usage++;
450
451	return devm_add_action_or_reset(uncore->dev,
452					devm_hisi_uncore_remove_platform_gov,
453					uncore);
454}
455
456/*
457 * Returns:
458 * 0 if success, uncore->related_cpus is set.
459 * -EINVAL if property not found, or property found but without elements in it,
460 * or invalid arguments received in any of the subroutine.
461 * Other error codes if it goes wrong.
462 */
463static int hisi_uncore_mark_related_cpus(struct hisi_uncore_freq *uncore,
464				 char *property, int (*get_topo_id)(int cpu),
465				 const struct cpumask *(*get_cpumask)(int cpu))
466{
467	unsigned int i, cpu;
468	size_t len;
469	int rc;
470
471	rc = device_property_count_u32(uncore->dev, property);
472	if (rc < 0)
473		return rc;
474	if (rc == 0)
475		return -EINVAL;
476
477	len = rc;
478	u32 *num __free(kfree) = kcalloc(len, sizeof(*num), GFP_KERNEL);
479	if (!num)
480		return -ENOMEM;
481
482	rc = device_property_read_u32_array(uncore->dev, property, num, len);
483	if (rc)
484		return rc;
485
486	for (i = 0; i < len; i++) {
487		for_each_possible_cpu(cpu) {
488			if (get_topo_id(cpu) != num[i])
489				continue;
490
491			cpumask_or(&uncore->related_cpus,
492				   &uncore->related_cpus, get_cpumask(cpu));
493			break;
494		}
495	}
496
497	return 0;
498}
499
500static int get_package_id(int cpu)
501{
502	return topology_physical_package_id(cpu);
503}
504
505static const struct cpumask *get_package_cpumask(int cpu)
506{
507	return topology_core_cpumask(cpu);
508}
509
510static int get_cluster_id(int cpu)
511{
512	return topology_cluster_id(cpu);
513}
514
515static const struct cpumask *get_cluster_cpumask(int cpu)
516{
517	return topology_cluster_cpumask(cpu);
518}
519
520static int hisi_uncore_mark_related_cpus_wrap(struct hisi_uncore_freq *uncore)
521{
522	int rc;
523
524	cpumask_clear(&uncore->related_cpus);
525
526	rc = hisi_uncore_mark_related_cpus(uncore, "related-package",
527					   get_package_id,
528					   get_package_cpumask);
529	/* Success, or firmware probably broken */
530	if (!rc || rc != -EINVAL)
531		return rc;
532
533	/* Try another property name if rc == -EINVAL */
534	return hisi_uncore_mark_related_cpus(uncore, "related-cluster",
535					     get_cluster_id,
536					     get_cluster_cpumask);
537}
538
539static ssize_t related_cpus_show(struct device *dev,
540				 struct device_attribute *attr, char *buf)
541{
542	struct hisi_uncore_freq *uncore = dev_get_drvdata(dev->parent);
543
544	return cpumap_print_to_pagebuf(true, buf, &uncore->related_cpus);
545}
546
547static DEVICE_ATTR_RO(related_cpus);
548
549static struct attribute *hisi_uncore_freq_attrs[] = {
550	&dev_attr_related_cpus.attr,
551	NULL
552};
553ATTRIBUTE_GROUPS(hisi_uncore_freq);
554
555static int hisi_uncore_devfreq_register(struct hisi_uncore_freq *uncore)
556{
557	struct devfreq_dev_profile *profile;
558	struct device *dev = uncore->dev;
559	unsigned long freq;
560	u32 data;
561	int rc;
562
563	rc = hisi_uncore_get_cur_freq(dev, &freq);
564	if (rc)
565		return dev_err_probe(dev, rc, "Failed to get plat init freq\n");
566
567	profile = devm_kzalloc(dev, sizeof(*profile), GFP_KERNEL);
568	if (!profile)
569		return -ENOMEM;
570
571	*profile = (struct devfreq_dev_profile) {
572		.initial_freq = freq,
573		.polling_ms = HUCF_DEFAULT_POLLING_MS,
574		.timer = DEVFREQ_TIMER_DELAYED,
575		.target = hisi_uncore_target,
576		.get_dev_status = hisi_uncore_get_dev_status,
577		.get_cur_freq = hisi_uncore_get_cur_freq,
578		.dev_groups = hisi_uncore_freq_groups,
579	};
580
581	rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_MODE, &data);
582	if (rc)
583		return dev_err_probe(dev, rc, "Failed to get operate mode\n");
584
585	if (data == HUCF_MODE_PLATFORM)
586		uncore->devfreq = devm_devfreq_add_device(dev, profile,
587					  hisi_platform_governor.name, NULL);
588	else
589		uncore->devfreq = devm_devfreq_add_device(dev, profile,
590					  DEVFREQ_GOV_PERFORMANCE, NULL);
591	if (IS_ERR(uncore->devfreq))
592		return dev_err_probe(dev, PTR_ERR(uncore->devfreq),
593			"Failed to add devfreq device\n");
594
595	return 0;
596}
597
598static int hisi_uncore_freq_probe(struct platform_device *pdev)
599{
600	struct hisi_uncore_freq *uncore;
601	struct device *dev = &pdev->dev;
602	u32 cap;
603	int rc;
604
605	uncore = devm_kzalloc(dev, sizeof(*uncore), GFP_KERNEL);
606	if (!uncore)
607		return -ENOMEM;
608
609	uncore->dev = dev;
610	platform_set_drvdata(pdev, uncore);
611
612	rc = hisi_uncore_init_pcc_chan(uncore);
613	if (rc)
614		return dev_err_probe(dev, rc, "Failed to init PCC channel\n");
615
616	rc = hisi_uncore_init_opp(uncore);
617	if (rc)
618		return dev_err_probe(dev, rc, "Failed to init OPP\n");
619
620	rc = hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_GET_CAP, &cap);
621	if (rc)
622		return dev_err_probe(dev, rc, "Failed to get capability\n");
623
624	uncore->cap = cap;
625
626	rc = hisi_uncore_add_platform_gov(uncore);
627	if (rc)
628		return dev_err_probe(dev, rc, "Failed to add hisi_platform governor\n");
629
630	rc = hisi_uncore_mark_related_cpus_wrap(uncore);
631	if (rc)
632		return dev_err_probe(dev, rc, "Failed to mark related cpus\n");
633
634	rc = hisi_uncore_devfreq_register(uncore);
635	if (rc)
636		return dev_err_probe(dev, rc, "Failed to register devfreq\n");
637
638	return 0;
639}
640
641static const struct acpi_device_id hisi_uncore_freq_acpi_match[] = {
642	{ "HISI04F1", },
643	{ }
644};
645MODULE_DEVICE_TABLE(acpi, hisi_uncore_freq_acpi_match);
646
647static struct platform_driver hisi_uncore_freq_drv = {
648	.probe	= hisi_uncore_freq_probe,
649	.driver = {
650		.name = "hisi_uncore_freq",
651		.acpi_match_table = hisi_uncore_freq_acpi_match,
652	},
653};
654module_platform_driver(hisi_uncore_freq_drv);
655
656MODULE_DESCRIPTION("HiSilicon uncore frequency scaling driver");
657MODULE_AUTHOR("Jie Zhan <zhanjie9@hisilicon.com>");
658MODULE_LICENSE("GPL");