uncore-frequency.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Intel Uncore Frequency Setting
  4 * Copyright (c) 2022, Intel Corporation.
  5 * All rights reserved.
  6 *
  7 * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
  8 * one control CPU is identified per die to read/write limit. This control CPU
  9 * is changed, if the CPU state is changed to offline. When the last CPU is
 10 * offline in a die then remove the sysfs object for that die.
 11 * The majority of actual code is related to sysfs create and read/write
 12 * attributes.
 13 *
 14 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
 15 */
 16
 17#include <linux/bitfield.h>
 18#include <linux/cpu.h>
 19#include <linux/module.h>
 20#include <linux/slab.h>
 21#include <linux/suspend.h>
 22#include <asm/cpu_device_id.h>
 23#include <asm/intel-family.h>
 24#include <asm/msr.h>
 25
 26#include "uncore-frequency-common.h"
 27
 28/* Max instances for uncore data, one for each die */
 29static int uncore_max_entries __read_mostly;
 30/* Storage for uncore data for all instances */
 31static struct uncore_data *uncore_instances;
 32/* Stores the CPU mask of the target CPUs to use during uncore read/write */
 33static cpumask_t uncore_cpu_mask;
 34/* CPU online callback register instance */
 35static enum cpuhp_state uncore_hp_state __read_mostly;
 36
 37#define MSR_UNCORE_RATIO_LIMIT	0x620
 38#define MSR_UNCORE_PERF_STATUS	0x621
 39#define UNCORE_FREQ_KHZ_MULTIPLIER	100000
 40
 41#define UNCORE_MAX_RATIO_MASK	GENMASK_ULL(6, 0)
 42#define UNCORE_MIN_RATIO_MASK	GENMASK_ULL(14, 8)
 43
 44#define UNCORE_CURRENT_RATIO_MASK	GENMASK_ULL(6, 0)
 45
 46static int uncore_read_control_freq(struct uncore_data *data, unsigned int *value,
 47				    enum uncore_index index)
 48{
 49	u64 cap;
 50	int ret;
 51
 52	if (data->control_cpu < 0)
 53		return -ENXIO;
 54
 55	ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
 56	if (ret)
 57		return ret;
 58
 59	if (index == UNCORE_INDEX_MAX_FREQ)
 60		*value = FIELD_GET(UNCORE_MAX_RATIO_MASK, cap) * UNCORE_FREQ_KHZ_MULTIPLIER;
 61	else
 62		*value = FIELD_GET(UNCORE_MIN_RATIO_MASK, cap) * UNCORE_FREQ_KHZ_MULTIPLIER;
 63
 64	return 0;
 65}
 66
 67static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
 68				     enum uncore_index index)
 69{
 70	int ret;
 71	u64 cap;
 72
 73	input /= UNCORE_FREQ_KHZ_MULTIPLIER;
 74	if (!input || input > FIELD_MAX(UNCORE_MAX_RATIO_MASK))
 75		return -EINVAL;
 76
 77	if (data->control_cpu < 0)
 78		return -ENXIO;
 79
 80	ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
 81	if (ret)
 82		return ret;
 83
 84	if (index == UNCORE_INDEX_MAX_FREQ) {
 85		cap &= ~UNCORE_MAX_RATIO_MASK;
 86		cap |= FIELD_PREP(UNCORE_MAX_RATIO_MASK, input);
 87	} else  {
 88		cap &= ~UNCORE_MIN_RATIO_MASK;
 89		cap |= FIELD_PREP(UNCORE_MIN_RATIO_MASK, input);
 90	}
 91
 92	ret = wrmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
 93	if (ret)
 94		return ret;
 95
 96	data->stored_uncore_data = cap;
 97
 98	return 0;
 99}
100
101static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
102{
103	u64 ratio;
104	int ret;
105
106	if (data->control_cpu < 0)
107		return -ENXIO;
108
109	ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_PERF_STATUS, &ratio);
110	if (ret)
111		return ret;
112
113	*freq = FIELD_GET(UNCORE_CURRENT_RATIO_MASK, ratio) * UNCORE_FREQ_KHZ_MULTIPLIER;
114
115	return 0;
116}
117
118static int uncore_read(struct uncore_data *data, unsigned int *value, enum uncore_index index)
119{
120	switch (index) {
121	case UNCORE_INDEX_MIN_FREQ:
122	case UNCORE_INDEX_MAX_FREQ:
123		return uncore_read_control_freq(data, value, index);
124
125	case UNCORE_INDEX_CURRENT_FREQ:
126		return uncore_read_freq(data, value);
127
128	default:
129		break;
130	}
131
132	return -EOPNOTSUPP;
133}
134
135/* Caller provides protection */
136static struct uncore_data *uncore_get_instance(unsigned int cpu)
137{
138	int id = topology_logical_die_id(cpu);
139
140	if (id >= 0 && id < uncore_max_entries)
141		return &uncore_instances[id];
142
143	return NULL;
144}
145
146static int uncore_event_cpu_online(unsigned int cpu)
147{
148	struct uncore_data *data;
149	int target;
150	int ret;
151
152	/* Check if there is an online cpu in the package for uncore MSR */
153	target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
154	if (target < nr_cpu_ids)
155		return 0;
156
157	data = uncore_get_instance(cpu);
158	if (!data)
159		return 0;
160
161	data->package_id = topology_physical_package_id(cpu);
162	data->die_id = topology_die_id(cpu);
163	data->domain_id = UNCORE_DOMAIN_ID_INVALID;
164
165	ret = uncore_freq_add_entry(data, cpu);
166	if (ret)
167		return ret;
168
169	/* Use this CPU on this die as a control CPU */
170	cpumask_set_cpu(cpu, &uncore_cpu_mask);
171
172	return 0;
173}
174
175static int uncore_event_cpu_offline(unsigned int cpu)
176{
177	struct uncore_data *data;
178	int target;
179
180	data = uncore_get_instance(cpu);
181	if (!data)
182		return 0;
183
184	/* Check if existing cpu is used for uncore MSRs */
185	if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
186		return 0;
187
188	/* Find a new cpu to set uncore MSR */
189	target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
190
191	if (target < nr_cpu_ids) {
192		cpumask_set_cpu(target, &uncore_cpu_mask);
193		uncore_freq_add_entry(data, target);
194	} else {
195		uncore_freq_remove_die_entry(data);
196	}
197
198	return 0;
199}
200
201static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
202			    void *_unused)
203{
204	int i;
205
206	switch (mode) {
207	case PM_POST_HIBERNATION:
208	case PM_POST_RESTORE:
209	case PM_POST_SUSPEND:
210		for (i = 0; i < uncore_max_entries; ++i) {
211			struct uncore_data *data = &uncore_instances[i];
212
213			if (!data || !data->valid || !data->stored_uncore_data)
214				return 0;
215
216			wrmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT,
217				      data->stored_uncore_data);
218		}
219		break;
220	default:
221		break;
222	}
223	return 0;
224}
225
226static struct notifier_block uncore_pm_nb = {
227	.notifier_call = uncore_pm_notify,
228};
229
230static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
231	X86_MATCH_VFM(INTEL_BROADWELL_G,	NULL),
232	X86_MATCH_VFM(INTEL_BROADWELL_X,	NULL),
233	X86_MATCH_VFM(INTEL_BROADWELL_D,	NULL),
234	X86_MATCH_VFM(INTEL_SKYLAKE_X,	NULL),
235	X86_MATCH_VFM(INTEL_ICELAKE_X,	NULL),
236	X86_MATCH_VFM(INTEL_ICELAKE_D,	NULL),
237	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL),
238	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, NULL),
239	X86_MATCH_VFM(INTEL_KABYLAKE, NULL),
240	X86_MATCH_VFM(INTEL_KABYLAKE_L, NULL),
241	X86_MATCH_VFM(INTEL_COMETLAKE, NULL),
242	X86_MATCH_VFM(INTEL_COMETLAKE_L, NULL),
243	X86_MATCH_VFM(INTEL_CANNONLAKE_L, NULL),
244	X86_MATCH_VFM(INTEL_ICELAKE, NULL),
245	X86_MATCH_VFM(INTEL_ICELAKE_L, NULL),
246	X86_MATCH_VFM(INTEL_ROCKETLAKE, NULL),
247	X86_MATCH_VFM(INTEL_TIGERLAKE, NULL),
248	X86_MATCH_VFM(INTEL_TIGERLAKE_L, NULL),
249	X86_MATCH_VFM(INTEL_ALDERLAKE, NULL),
250	X86_MATCH_VFM(INTEL_ALDERLAKE_L, NULL),
251	X86_MATCH_VFM(INTEL_RAPTORLAKE, NULL),
252	X86_MATCH_VFM(INTEL_RAPTORLAKE_P, NULL),
253	X86_MATCH_VFM(INTEL_RAPTORLAKE_S, NULL),
254	X86_MATCH_VFM(INTEL_METEORLAKE, NULL),
255	X86_MATCH_VFM(INTEL_METEORLAKE_L, NULL),
256	X86_MATCH_VFM(INTEL_ARROWLAKE, NULL),
257	X86_MATCH_VFM(INTEL_ARROWLAKE_H, NULL),
258	X86_MATCH_VFM(INTEL_LUNARLAKE_M, NULL),
259	X86_MATCH_VFM(INTEL_PANTHERLAKE_L, NULL),
260	X86_MATCH_VFM(INTEL_WILDCATLAKE_L, NULL),
261	X86_MATCH_VFM(INTEL_NOVALAKE, NULL),
262	X86_MATCH_VFM(INTEL_NOVALAKE_L, NULL),
263	{}
264};
265MODULE_DEVICE_TABLE(x86cpu, intel_uncore_cpu_ids);
266
267static int __init intel_uncore_init(void)
268{
269	const struct x86_cpu_id *id;
270	int ret;
271
272	if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
273		return -ENODEV;
274
275	id = x86_match_cpu(intel_uncore_cpu_ids);
276	if (!id)
277		return -ENODEV;
278
279	uncore_max_entries = topology_max_packages() *
280					topology_max_dies_per_package();
281	uncore_instances = kcalloc(uncore_max_entries,
282				   sizeof(*uncore_instances), GFP_KERNEL);
283	if (!uncore_instances)
284		return -ENOMEM;
285
286	ret = uncore_freq_common_init(uncore_read, uncore_write_control_freq);
287	if (ret)
288		goto err_free;
289
290	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
291				"platform/x86/uncore-freq:online",
292				uncore_event_cpu_online,
293				uncore_event_cpu_offline);
294	if (ret < 0)
295		goto err_rem_kobj;
296
297	uncore_hp_state = ret;
298
299	ret = register_pm_notifier(&uncore_pm_nb);
300	if (ret)
301		goto err_rem_state;
302
303	return 0;
304
305err_rem_state:
306	cpuhp_remove_state(uncore_hp_state);
307err_rem_kobj:
308	uncore_freq_common_exit();
309err_free:
310	kfree(uncore_instances);
311
312	return ret;
313}
314module_init(intel_uncore_init)
315
316static void __exit intel_uncore_exit(void)
317{
318	int i;
319
320	unregister_pm_notifier(&uncore_pm_nb);
321	cpuhp_remove_state(uncore_hp_state);
322	for (i = 0; i < uncore_max_entries; ++i)
323		uncore_freq_remove_die_entry(&uncore_instances[i]);
324	uncore_freq_common_exit();
325	kfree(uncore_instances);
326}
327module_exit(intel_uncore_exit)
328
329MODULE_IMPORT_NS("INTEL_UNCORE_FREQUENCY");
330MODULE_LICENSE("GPL v2");
331MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");