hfi.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * AMD Hardware Feedback Interface Driver
  4 *
  5 * Copyright (C) 2025 Advanced Micro Devices, Inc. All Rights Reserved.
  6 *
  7 * Authors: Perry Yuan <Perry.Yuan@amd.com>
  8 *          Mario Limonciello <mario.limonciello@amd.com>
  9 */
 10
 11#define pr_fmt(fmt)  "amd-hfi: " fmt
 12
 13#include <linux/acpi.h>
 14#include <linux/cpu.h>
 15#include <linux/debugfs.h>
 16#include <linux/gfp.h>
 17#include <linux/init.h>
 18#include <linux/io.h>
 19#include <linux/kernel.h>
 20#include <linux/module.h>
 21#include <linux/mailbox_client.h>
 22#include <linux/mutex.h>
 23#include <linux/percpu-defs.h>
 24#include <linux/platform_device.h>
 25#include <linux/smp.h>
 26#include <linux/topology.h>
 27#include <linux/workqueue.h>
 28
 29#include <asm/cpu_device_id.h>
 30
 31#include <acpi/pcc.h>
 32#include <acpi/cppc_acpi.h>
 33
 34#define AMD_HFI_DRIVER		"amd_hfi"
 35#define AMD_HFI_MAILBOX_COUNT		1
 36#define AMD_HETERO_RANKING_TABLE_VER	2
 37
 38#define AMD_HETERO_CPUID_27	0x80000027
 39
 40static struct platform_device *device;
 41
 42/**
 43 * struct amd_shmem_info - Shared memory table for AMD HFI
 44 *
 45 * @header:	The PCCT table header including signature, length flags and command.
 46 * @version_number:		Version number of the table
 47 * @n_logical_processors:	Number of logical processors
 48 * @n_capabilities:		Number of ranking dimensions (performance, efficiency, etc)
 49 * @table_update_context:	Command being sent over the subspace
 50 * @n_bitmaps:			Number of 32-bit bitmaps to enumerate all the APIC IDs
 51 *				This is based on the maximum APIC ID enumerated in the system
 52 * @reserved:			24 bit spare
 53 * @table_data:			Bit Map(s) of enabled logical processors
 54 *				Followed by the ranking data for each logical processor
 55 */
 56struct amd_shmem_info {
 57	struct acpi_pcct_ext_pcc_shared_memory header;
 58	u32	version_number		:8,
 59		n_logical_processors	:8,
 60		n_capabilities		:8,
 61		table_update_context	:8;
 62	u32	n_bitmaps		:8,
 63		reserved		:24;
 64	u32	table_data[];
 65};
 66
 67struct amd_hfi_data {
 68	const char	*name;
 69	struct device	*dev;
 70
 71	/* PCCT table related */
 72	struct pcc_mbox_chan	*pcc_chan;
 73	void __iomem		*pcc_comm_addr;
 74	struct acpi_subtable_header	*pcct_entry;
 75	struct amd_shmem_info	*shmem;
 76
 77	struct dentry *dbgfs_dir;
 78};
 79
 80/**
 81 * struct amd_hfi_classes - HFI class capabilities per CPU
 82 * @perf:	Performance capability
 83 * @eff:	Power efficiency capability
 84 *
 85 * Capabilities of a logical processor in the ranking table. These capabilities
 86 * are unitless and specific to each HFI class.
 87 */
 88struct amd_hfi_classes {
 89	u32	perf;
 90	u32	eff;
 91};
 92
 93/**
 94 * struct amd_hfi_cpuinfo - HFI workload class info per CPU
 95 * @cpu:		CPU index
 96 * @apic_id:		APIC id of the current CPU
 97 * @class_index:	workload class ID index
 98 * @nr_class:		max number of workload class supported
 99 * @ipcc_scores:	ipcc scores for each class
100 * @amd_hfi_classes:	current CPU workload class ranking data
101 *
102 * Parameters of a logical processor linked with hardware feedback class.
103 */
104struct amd_hfi_cpuinfo {
105	int		cpu;
106	u32		apic_id;
107	s16		class_index;
108	u8		nr_class;
109	int		*ipcc_scores;
110	struct amd_hfi_classes	*amd_hfi_classes;
111};
112
113static DEFINE_PER_CPU(struct amd_hfi_cpuinfo, amd_hfi_cpuinfo) = {.class_index = -1};
114
115static DEFINE_MUTEX(hfi_cpuinfo_lock);
116
117static void amd_hfi_sched_itmt_work(struct work_struct *work)
118{
119	sched_set_itmt_support();
120}
121static DECLARE_WORK(sched_amd_hfi_itmt_work, amd_hfi_sched_itmt_work);
122
123static int find_cpu_index_by_apicid(unsigned int target_apicid)
124{
125	int cpu_index;
126
127	for_each_possible_cpu(cpu_index) {
128		struct cpuinfo_x86 *info = &cpu_data(cpu_index);
129
130		if (info->topo.apicid == target_apicid) {
131			pr_debug("match APIC id %u for CPU index: %d\n",
132				 info->topo.apicid, cpu_index);
133			return cpu_index;
134		}
135	}
136
137	return -ENODEV;
138}
139
140static int amd_hfi_fill_metadata(struct amd_hfi_data *amd_hfi_data)
141{
142	struct acpi_pcct_ext_pcc_slave *pcct_ext =
143		(struct acpi_pcct_ext_pcc_slave *)amd_hfi_data->pcct_entry;
144	void __iomem *pcc_comm_addr;
145	u32 apic_start = 0;
146
147	pcc_comm_addr = acpi_os_ioremap(amd_hfi_data->pcc_chan->shmem_base_addr,
148					amd_hfi_data->pcc_chan->shmem_size);
149	if (!pcc_comm_addr) {
150		dev_err(amd_hfi_data->dev, "failed to ioremap PCC common region mem\n");
151		return -ENOMEM;
152	}
153
154	memcpy_fromio(amd_hfi_data->shmem, pcc_comm_addr, pcct_ext->length);
155	iounmap(pcc_comm_addr);
156
157	if (amd_hfi_data->shmem->header.signature != PCC_SIGNATURE) {
158		dev_err(amd_hfi_data->dev, "invalid signature in shared memory\n");
159		return -EINVAL;
160	}
161	if (amd_hfi_data->shmem->version_number != AMD_HETERO_RANKING_TABLE_VER) {
162		dev_err(amd_hfi_data->dev, "invalid version %d\n",
163			amd_hfi_data->shmem->version_number);
164		return -EINVAL;
165	}
166
167	for (unsigned int i = 0; i < amd_hfi_data->shmem->n_bitmaps; i++) {
168		u32 bitmap = amd_hfi_data->shmem->table_data[i];
169
170		for (unsigned int j = 0; j < BITS_PER_TYPE(u32); j++) {
171			u32 apic_id = i * BITS_PER_TYPE(u32) + j;
172			struct amd_hfi_cpuinfo *info;
173			int cpu_index, apic_index;
174
175			if (!(bitmap & BIT(j)))
176				continue;
177
178			cpu_index = find_cpu_index_by_apicid(apic_id);
179			if (cpu_index < 0) {
180				dev_warn(amd_hfi_data->dev, "APIC ID %u not found\n", apic_id);
181				continue;
182			}
183
184			info = per_cpu_ptr(&amd_hfi_cpuinfo, cpu_index);
185			info->apic_id = apic_id;
186
187			/* Fill the ranking data for each logical processor */
188			info = per_cpu_ptr(&amd_hfi_cpuinfo, cpu_index);
189			apic_index = apic_start * info->nr_class * 2;
190			for (unsigned int k = 0; k < info->nr_class; k++) {
191				u32 *table = amd_hfi_data->shmem->table_data +
192					     amd_hfi_data->shmem->n_bitmaps +
193					     i * info->nr_class;
194
195				info->amd_hfi_classes[k].eff = table[apic_index + 2 * k];
196				info->amd_hfi_classes[k].perf = table[apic_index + 2 * k + 1];
197			}
198			apic_start++;
199		}
200	}
201
202	return 0;
203}
204
205static int amd_hfi_alloc_class_data(struct platform_device *pdev)
206{
207	struct amd_hfi_cpuinfo *hfi_cpuinfo;
208	struct device *dev = &pdev->dev;
209	u32 nr_class_id;
210	int idx;
211
212	nr_class_id = cpuid_eax(AMD_HETERO_CPUID_27);
213	if (nr_class_id > 255) {
214		dev_err(dev, "number of supported classes too large: %d\n",
215			nr_class_id);
216		return -EINVAL;
217	}
218
219	for_each_possible_cpu(idx) {
220		struct amd_hfi_classes *classes;
221		int *ipcc_scores;
222
223		classes = devm_kcalloc(dev,
224				       nr_class_id,
225				       sizeof(struct amd_hfi_classes),
226				       GFP_KERNEL);
227		if (!classes)
228			return -ENOMEM;
229		ipcc_scores = devm_kcalloc(dev, nr_class_id, sizeof(int), GFP_KERNEL);
230		if (!ipcc_scores)
231			return -ENOMEM;
232		hfi_cpuinfo = per_cpu_ptr(&amd_hfi_cpuinfo, idx);
233		hfi_cpuinfo->amd_hfi_classes = classes;
234		hfi_cpuinfo->ipcc_scores = ipcc_scores;
235		hfi_cpuinfo->nr_class = nr_class_id;
236	}
237
238	return 0;
239}
240
241static void amd_hfi_remove(struct platform_device *pdev)
242{
243	struct amd_hfi_data *dev = platform_get_drvdata(pdev);
244
245	debugfs_remove_recursive(dev->dbgfs_dir);
246}
247
248static int amd_set_hfi_ipcc_score(struct amd_hfi_cpuinfo *hfi_cpuinfo, int cpu)
249{
250	for (int i = 0; i < hfi_cpuinfo->nr_class; i++)
251		WRITE_ONCE(hfi_cpuinfo->ipcc_scores[i],
252			   hfi_cpuinfo->amd_hfi_classes[i].perf);
253
254	sched_set_itmt_core_prio(hfi_cpuinfo->ipcc_scores[0], cpu);
255
256	return 0;
257}
258
259static int amd_hfi_set_state(unsigned int cpu, bool state)
260{
261	int ret;
262
263	ret = wrmsrq_on_cpu(cpu, MSR_AMD_WORKLOAD_CLASS_CONFIG, state ? 1 : 0);
264	if (ret)
265		return ret;
266
267	return wrmsrq_on_cpu(cpu, MSR_AMD_WORKLOAD_HRST, 0x1);
268}
269
270/**
271 * amd_hfi_online() - Enable workload classification on @cpu
272 * @cpu: CPU in which the workload classification will be enabled
273 *
274 * Return: 0 on success, negative error code on failure.
275 */
276static int amd_hfi_online(unsigned int cpu)
277{
278	struct amd_hfi_cpuinfo *hfi_info = per_cpu_ptr(&amd_hfi_cpuinfo, cpu);
279	struct amd_hfi_classes *hfi_classes;
280	int ret;
281
282	if (WARN_ON_ONCE(!hfi_info))
283		return -EINVAL;
284
285	/*
286	 * Check if @cpu as an associated, initialized and ranking data must
287	 * be filled.
288	 */
289	hfi_classes = hfi_info->amd_hfi_classes;
290	if (!hfi_classes)
291		return -EINVAL;
292
293	guard(mutex)(&hfi_cpuinfo_lock);
294
295	ret = amd_hfi_set_state(cpu, true);
296	if (ret)
297		pr_err("WCT enable failed for CPU %u\n", cpu);
298
299	return ret;
300}
301
302/**
303 * amd_hfi_offline() - Disable workload classification on @cpu
304 * @cpu: CPU in which the workload classification will be disabled
305 *
306 * Remove @cpu from those covered by its HFI instance.
307 *
308 * Return: 0 on success, negative error code on failure
309 */
310static int amd_hfi_offline(unsigned int cpu)
311{
312	struct amd_hfi_cpuinfo *hfi_info = &per_cpu(amd_hfi_cpuinfo, cpu);
313	int ret;
314
315	if (WARN_ON_ONCE(!hfi_info))
316		return -EINVAL;
317
318	guard(mutex)(&hfi_cpuinfo_lock);
319
320	ret = amd_hfi_set_state(cpu, false);
321	if (ret)
322		pr_err("WCT disable failed for CPU %u\n", cpu);
323
324	return ret;
325}
326
327static int update_hfi_ipcc_scores(void)
328{
329	int cpu;
330	int ret;
331
332	for_each_possible_cpu(cpu) {
333		struct amd_hfi_cpuinfo *hfi_cpuinfo = per_cpu_ptr(&amd_hfi_cpuinfo, cpu);
334
335		ret = amd_set_hfi_ipcc_score(hfi_cpuinfo, cpu);
336		if (ret)
337			return ret;
338	}
339
340	return 0;
341}
342
343static int amd_hfi_metadata_parser(struct platform_device *pdev,
344				   struct amd_hfi_data *amd_hfi_data)
345{
346	struct acpi_pcct_ext_pcc_slave *pcct_ext;
347	struct acpi_subtable_header *pcct_entry;
348	struct mbox_chan *pcc_mbox_channels;
349	struct acpi_table_header *pcct_tbl;
350	struct pcc_mbox_chan *pcc_chan;
351	acpi_status status;
352	int ret;
353
354	pcc_mbox_channels = devm_kcalloc(&pdev->dev, AMD_HFI_MAILBOX_COUNT,
355					 sizeof(*pcc_mbox_channels), GFP_KERNEL);
356	if (!pcc_mbox_channels)
357		return -ENOMEM;
358
359	pcc_chan = devm_kcalloc(&pdev->dev, AMD_HFI_MAILBOX_COUNT,
360				sizeof(*pcc_chan), GFP_KERNEL);
361	if (!pcc_chan)
362		return -ENOMEM;
363
364	status = acpi_get_table(ACPI_SIG_PCCT, 0, &pcct_tbl);
365	if (ACPI_FAILURE(status) || !pcct_tbl)
366		return -ENODEV;
367
368	/* get pointer to the first PCC subspace entry */
369	pcct_entry = (struct acpi_subtable_header *) (
370			(unsigned long)pcct_tbl + sizeof(struct acpi_table_pcct));
371
372	pcc_chan->mchan = &pcc_mbox_channels[0];
373
374	amd_hfi_data->pcc_chan = pcc_chan;
375	amd_hfi_data->pcct_entry = pcct_entry;
376	pcct_ext = (struct acpi_pcct_ext_pcc_slave *)pcct_entry;
377
378	if (pcct_ext->length <= 0) {
379		ret = -EINVAL;
380		goto out;
381	}
382
383	amd_hfi_data->shmem = devm_kzalloc(amd_hfi_data->dev, pcct_ext->length, GFP_KERNEL);
384	if (!amd_hfi_data->shmem) {
385		ret = -ENOMEM;
386		goto out;
387	}
388
389	pcc_chan->shmem_base_addr = pcct_ext->base_address;
390	pcc_chan->shmem_size = pcct_ext->length;
391
392	/* parse the shared memory info from the PCCT table */
393	ret = amd_hfi_fill_metadata(amd_hfi_data);
394
395out:
396	/* Don't leak any ACPI memory */
397	acpi_put_table(pcct_tbl);
398
399	return ret;
400}
401
402static int class_capabilities_show(struct seq_file *s, void *unused)
403{
404	u32 cpu, idx;
405
406	seq_puts(s, "CPU #\tWLC\tPerf\tEff\n");
407	for_each_possible_cpu(cpu) {
408		struct amd_hfi_cpuinfo *hfi_cpuinfo = per_cpu_ptr(&amd_hfi_cpuinfo, cpu);
409
410		seq_printf(s, "%d", cpu);
411		for (idx = 0; idx < hfi_cpuinfo->nr_class; idx++) {
412			seq_printf(s, "\t%u\t%u\t%u\n", idx,
413				   hfi_cpuinfo->amd_hfi_classes[idx].perf,
414				   hfi_cpuinfo->amd_hfi_classes[idx].eff);
415		}
416	}
417
418	return 0;
419}
420DEFINE_SHOW_ATTRIBUTE(class_capabilities);
421
422static int amd_hfi_pm_resume(struct device *dev)
423{
424	int ret, cpu;
425
426	for_each_online_cpu(cpu) {
427		ret = amd_hfi_set_state(cpu, true);
428		if (ret < 0) {
429			dev_err(dev, "failed to enable workload class config: %d\n", ret);
430			return ret;
431		}
432	}
433
434	return 0;
435}
436
437static int amd_hfi_pm_suspend(struct device *dev)
438{
439	int ret, cpu;
440
441	for_each_online_cpu(cpu) {
442		ret = amd_hfi_set_state(cpu, false);
443		if (ret < 0) {
444			dev_err(dev, "failed to disable workload class config: %d\n", ret);
445			return ret;
446		}
447	}
448
449	return 0;
450}
451
452static DEFINE_SIMPLE_DEV_PM_OPS(amd_hfi_pm_ops, amd_hfi_pm_suspend, amd_hfi_pm_resume);
453
454static const struct acpi_device_id amd_hfi_platform_match[] = {
455	{"AMDI0104", 0},
456	{ }
457};
458MODULE_DEVICE_TABLE(acpi, amd_hfi_platform_match);
459
460static int amd_hfi_probe(struct platform_device *pdev)
461{
462	struct amd_hfi_data *amd_hfi_data;
463	int ret;
464
465	if (!acpi_match_device(amd_hfi_platform_match, &pdev->dev))
466		return -ENODEV;
467
468	amd_hfi_data = devm_kzalloc(&pdev->dev, sizeof(*amd_hfi_data), GFP_KERNEL);
469	if (!amd_hfi_data)
470		return -ENOMEM;
471
472	amd_hfi_data->dev = &pdev->dev;
473	platform_set_drvdata(pdev, amd_hfi_data);
474
475	ret = amd_hfi_alloc_class_data(pdev);
476	if (ret)
477		return ret;
478
479	ret = amd_hfi_metadata_parser(pdev, amd_hfi_data);
480	if (ret)
481		return ret;
482
483	ret = update_hfi_ipcc_scores();
484	if (ret)
485		return ret;
486
487	/*
488	 * Tasks will already be running at the time this happens. This is
489	 * OK because rankings will be adjusted by the callbacks.
490	 */
491	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/amd_hfi:online",
492				amd_hfi_online, amd_hfi_offline);
493	if (ret < 0)
494		return ret;
495
496	schedule_work(&sched_amd_hfi_itmt_work);
497
498	amd_hfi_data->dbgfs_dir = debugfs_create_dir("amd_hfi", arch_debugfs_dir);
499	debugfs_create_file("class_capabilities", 0644, amd_hfi_data->dbgfs_dir, pdev,
500			    &class_capabilities_fops);
501
502	return 0;
503}
504
505static struct platform_driver amd_hfi_driver = {
506	.driver = {
507		.name = AMD_HFI_DRIVER,
508		.pm = &amd_hfi_pm_ops,
509		.acpi_match_table = ACPI_PTR(amd_hfi_platform_match),
510	},
511	.probe = amd_hfi_probe,
512	.remove = amd_hfi_remove,
513};
514
515static int __init amd_hfi_init(void)
516{
517	int ret;
518
519	if (acpi_disabled ||
520	    !cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES) ||
521	    !cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS))
522		return -ENODEV;
523
524	device = platform_device_register_simple(AMD_HFI_DRIVER, -1, NULL, 0);
525	if (IS_ERR(device)) {
526		pr_err("unable to register HFI platform device\n");
527		return PTR_ERR(device);
528	}
529
530	ret = platform_driver_register(&amd_hfi_driver);
531	if (ret)
532		pr_err("failed to register HFI driver\n");
533
534	return ret;
535}
536
537static __exit void amd_hfi_exit(void)
538{
539	platform_driver_unregister(&amd_hfi_driver);
540	platform_device_unregister(device);
541}
542module_init(amd_hfi_init);
543module_exit(amd_hfi_exit);
544
545MODULE_LICENSE("GPL");
546MODULE_DESCRIPTION("AMD Hardware Feedback Interface Driver");