uncore-frequency-tpmi.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * uncore-frquency-tpmi: Uncore frequency scaling using TPMI
  4 *
  5 * Copyright (c) 2023, Intel Corporation.
  6 * All Rights Reserved.
  7 *
  8 * The hardware interface to read/write is basically substitution of
  9 * MSR 0x620 and 0x621.
 10 * There are specific MMIO offset and bits to get/set minimum and
 11 * maximum uncore ratio, similar to MSRs.
 12 * The scope of the uncore MSRs was package scope. But TPMI allows
 13 * new gen CPUs to have multiple uncore controls at uncore-cluster
 14 * level. Each package can have multiple power domains which further
 15 * can have multiple clusters.
 16 * Here number of power domains = number of resources in this aux
 17 * device. There are offsets and bits to discover number of clusters
 18 * and offset for each cluster level controls.
 19 *
 20 */
 21
 22#include <linux/auxiliary_bus.h>
 23#include <linux/bitfield.h>
 24#include <linux/bits.h>
 25#include <linux/intel_tpmi.h>
 26#include <linux/intel_vsec.h>
 27#include <linux/io.h>
 28#include <linux/module.h>
 29
 30#include "../tpmi_power_domains.h"
 31#include "uncore-frequency-common.h"
 32
 33#define	UNCORE_MAJOR_VERSION		0
 34#define	UNCORE_MINOR_VERSION		2
 35#define UNCORE_ELC_SUPPORTED_VERSION	2
 36#define UNCORE_HEADER_INDEX		0
 37#define UNCORE_FABRIC_CLUSTER_OFFSET	8
 38
 39/* status + control + adv_ctl1 + adv_ctl2 */
 40#define UNCORE_FABRIC_CLUSTER_SIZE	(4 * 8)
 41
 42#define UNCORE_STATUS_INDEX		0
 43#define UNCORE_CONTROL_INDEX		8
 44
 45#define UNCORE_FREQ_KHZ_MULTIPLIER	100000
 46
 47struct tpmi_uncore_struct;
 48
 49/* Information for each cluster */
 50struct tpmi_uncore_cluster_info {
 51	bool root_domain;
 52	bool elc_supported;
 53	u8 __iomem *cluster_base;
 54	u16 cdie_id;
 55	struct uncore_data uncore_data;
 56	struct tpmi_uncore_struct *uncore_root;
 57};
 58
 59/* Information for each power domain */
 60struct tpmi_uncore_power_domain_info {
 61	u8 __iomem *uncore_base;
 62	int ufs_header_ver;
 63	int cluster_count;
 64	struct tpmi_uncore_cluster_info *cluster_infos;
 65};
 66
 67/* Information for all power domains in a package */
 68struct tpmi_uncore_struct {
 69	int power_domain_count;
 70	int max_ratio;
 71	int min_ratio;
 72	struct tpmi_uncore_power_domain_info *pd_info;
 73	struct tpmi_uncore_cluster_info root_cluster;
 74	bool write_blocked;
 75};
 76
 77/* Bit definitions for STATUS register */
 78#define UNCORE_CURRENT_RATIO_MASK			GENMASK_ULL(6, 0)
 79
 80/* Bit definitions for CONTROL register */
 81#define UNCORE_MAX_RATIO_MASK				GENMASK_ULL(14, 8)
 82#define UNCORE_MIN_RATIO_MASK				GENMASK_ULL(21, 15)
 83#define UNCORE_EFF_LAT_CTRL_RATIO_MASK			GENMASK_ULL(28, 22)
 84#define UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK		GENMASK_ULL(38, 32)
 85#define UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE	BIT(39)
 86#define UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK		GENMASK_ULL(46, 40)
 87
 88/* Helper function to read MMIO offset for max/min control frequency */
 89static void read_control_freq(struct tpmi_uncore_cluster_info *cluster_info,
 90			     unsigned int *value, enum uncore_index index)
 91{
 92	u64 control;
 93
 94	control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
 95	if (index == UNCORE_INDEX_MAX_FREQ)
 96		*value = FIELD_GET(UNCORE_MAX_RATIO_MASK, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
 97	else
 98		*value = FIELD_GET(UNCORE_MIN_RATIO_MASK, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
 99}
100
101/* Helper function to read efficiency latency control values over MMIO */
102static int read_eff_lat_ctrl(struct uncore_data *data, unsigned int *val, enum uncore_index index)
103{
104	struct tpmi_uncore_cluster_info *cluster_info;
105	u64 ctrl;
106
107	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
108	if (cluster_info->root_domain)
109		return -ENODATA;
110
111	if (!cluster_info->elc_supported)
112		return -EOPNOTSUPP;
113
114	ctrl = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
115
116	switch (index) {
117	case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD:
118		*val = FIELD_GET(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK, ctrl);
119		*val *= 100;
120		*val = DIV_ROUND_UP(*val, FIELD_MAX(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK));
121		break;
122
123	case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD:
124		*val = FIELD_GET(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK, ctrl);
125		*val *= 100;
126		*val = DIV_ROUND_UP(*val, FIELD_MAX(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK));
127		break;
128
129	case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE:
130		*val = FIELD_GET(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE, ctrl);
131		break;
132	case UNCORE_INDEX_EFF_LAT_CTRL_FREQ:
133		*val = FIELD_GET(UNCORE_EFF_LAT_CTRL_RATIO_MASK, ctrl) * UNCORE_FREQ_KHZ_MULTIPLIER;
134		break;
135
136	default:
137		return -EOPNOTSUPP;
138	}
139
140	return 0;
141}
142
143#define UNCORE_MAX_RATIO	FIELD_MAX(UNCORE_MAX_RATIO_MASK)
144
145/* Helper for sysfs read for max/min frequencies. Called under mutex locks */
146static int uncore_read_control_freq(struct uncore_data *data, unsigned int *value,
147				    enum uncore_index index)
148{
149	struct tpmi_uncore_cluster_info *cluster_info;
150
151	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
152
153	if (cluster_info->root_domain) {
154		struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
155		unsigned int min, max, v;
156		int i;
157
158		min = UNCORE_MAX_RATIO * UNCORE_FREQ_KHZ_MULTIPLIER;
159		max = 0;
160
161		/*
162		 * Get the max/min by looking at each cluster. Get the lowest
163		 * min and highest max.
164		 */
165		for (i = 0; i < uncore_root->power_domain_count; ++i) {
166			int j;
167
168			for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) {
169				read_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
170						  &v, index);
171				if (v < min)
172					min = v;
173				if (v > max)
174					max = v;
175			}
176		}
177
178		if (index == UNCORE_INDEX_MIN_FREQ)
179			*value = min;
180		else
181			*value = max;
182
183		return 0;
184	}
185
186	read_control_freq(cluster_info, value, index);
187
188	return 0;
189}
190
191/* Helper function for writing efficiency latency control values over MMIO */
192static int write_eff_lat_ctrl(struct uncore_data *data, unsigned int val, enum uncore_index index)
193{
194	struct tpmi_uncore_cluster_info *cluster_info;
195	struct tpmi_uncore_struct *uncore_root;
196	u64 control;
197
198	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
199	uncore_root = cluster_info->uncore_root;
200
201	if (uncore_root->write_blocked)
202		return -EPERM;
203
204	if (cluster_info->root_domain)
205		return -ENODATA;
206
207	if (!cluster_info->elc_supported)
208		return -EOPNOTSUPP;
209
210	switch (index) {
211	case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD:
212		if (val > 100)
213			return -EINVAL;
214		break;
215
216	case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD:
217		if (val > 100)
218			return -EINVAL;
219		break;
220
221	case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE:
222		if (val > 1)
223			return -EINVAL;
224		break;
225
226	case UNCORE_INDEX_EFF_LAT_CTRL_FREQ:
227		val /= UNCORE_FREQ_KHZ_MULTIPLIER;
228		if (val > FIELD_MAX(UNCORE_EFF_LAT_CTRL_RATIO_MASK))
229			return -EINVAL;
230		break;
231
232	default:
233		return -EOPNOTSUPP;
234	}
235
236	control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
237
238	switch (index) {
239	case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD:
240		val *= FIELD_MAX(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK);
241		val /= 100;
242		control &= ~UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK;
243		control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_LOW_THRESHOLD_MASK, val);
244		break;
245
246	case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD:
247		val *= FIELD_MAX(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK);
248		val /= 100;
249		control &= ~UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK;
250		control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_MASK, val);
251		break;
252
253	case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE:
254		control &= ~UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE;
255		control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE, val);
256		break;
257
258	case UNCORE_INDEX_EFF_LAT_CTRL_FREQ:
259		control &= ~UNCORE_EFF_LAT_CTRL_RATIO_MASK;
260		control |= FIELD_PREP(UNCORE_EFF_LAT_CTRL_RATIO_MASK, val);
261		break;
262
263	default:
264		break;
265	}
266
267	writeq(control, cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
268
269	return 0;
270}
271
272/* Helper function to write MMIO offset for max/min control frequency */
273static void write_control_freq(struct tpmi_uncore_cluster_info *cluster_info, unsigned int input,
274			      unsigned int index)
275{
276	u64 control;
277
278	control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
279
280	if (index == UNCORE_INDEX_MAX_FREQ) {
281		control &= ~UNCORE_MAX_RATIO_MASK;
282		control |= FIELD_PREP(UNCORE_MAX_RATIO_MASK, input);
283	} else {
284		control &= ~UNCORE_MIN_RATIO_MASK;
285		control |= FIELD_PREP(UNCORE_MIN_RATIO_MASK, input);
286	}
287
288	writeq(control, (cluster_info->cluster_base + UNCORE_CONTROL_INDEX));
289}
290
291/* Helper for sysfs write for max/min frequencies. Called under mutex locks */
292static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
293				     enum uncore_index index)
294{
295	struct tpmi_uncore_cluster_info *cluster_info;
296	struct tpmi_uncore_struct *uncore_root;
297
298	input /= UNCORE_FREQ_KHZ_MULTIPLIER;
299	if (!input || input > UNCORE_MAX_RATIO)
300		return -EINVAL;
301
302	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
303	uncore_root = cluster_info->uncore_root;
304
305	if (uncore_root->write_blocked)
306		return -EPERM;
307
308	/* Update each cluster in a package */
309	if (cluster_info->root_domain) {
310		struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
311		int i;
312
313		for (i = 0; i < uncore_root->power_domain_count; ++i) {
314			int j;
315
316			for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j)
317				write_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
318						  input, index);
319		}
320
321		if (index == UNCORE_INDEX_MAX_FREQ)
322			uncore_root->max_ratio = input;
323		else
324			uncore_root->min_ratio = input;
325
326		return 0;
327	}
328
329	if (index == UNCORE_INDEX_MAX_FREQ && uncore_root->max_ratio &&
330	    uncore_root->max_ratio < input)
331		return -EINVAL;
332
333	if (index == UNCORE_INDEX_MIN_FREQ && uncore_root->min_ratio &&
334	    uncore_root->min_ratio > input)
335		return -EINVAL;
336
337	write_control_freq(cluster_info, input, index);
338
339	return 0;
340}
341
342/* Helper for sysfs read for the current uncore frequency. Called under mutex locks */
343static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
344{
345	struct tpmi_uncore_cluster_info *cluster_info;
346	u64 status;
347
348	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
349	if (cluster_info->root_domain)
350		return -ENODATA;
351
352	status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX);
353	*freq = FIELD_GET(UNCORE_CURRENT_RATIO_MASK, status) * UNCORE_FREQ_KHZ_MULTIPLIER;
354
355	return 0;
356}
357
358/*
359 * Agent types as per the TPMI UFS Specification for UFS_STATUS
360 * Agent Type - Core	Bit: 23
361 * Agent Type - Cache	Bit: 24
362 * Agent Type - Memory	Bit: 25
363 * Agent Type - IO	Bit: 26
364 */
365
366#define UNCORE_AGENT_TYPES	GENMASK_ULL(26, 23)
367
368/* Helper function to read agent type over MMIO and set the agent type mask */
369static void uncore_set_agent_type(struct tpmi_uncore_cluster_info *cluster_info)
370{
371	u64 status;
372
373	status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX);
374	cluster_info->uncore_data.agent_type_mask = FIELD_GET(UNCORE_AGENT_TYPES, status);
375}
376
377#define MAX_PARTITIONS	2
378
379/* IO domain ID start index for a partition */
380static u8 io_die_start[MAX_PARTITIONS];
381
382/* Next IO domain ID index after the current partition IO die IDs */
383static u8 io_die_index_next;
384
385/* Lock to protect io_die_start, io_die_index_next */
386static DEFINE_MUTEX(domain_lock);
387
388static void set_domain_id(int id,  int num_resources,
389			  struct oobmsm_plat_info *plat_info,
390			  struct tpmi_uncore_cluster_info *cluster_info)
391{
392	u8 part_io_index, cdie_range, pkg_io_index, max_dies;
393
394	if (plat_info->partition >= MAX_PARTITIONS) {
395		cluster_info->uncore_data.domain_id = id;
396		return;
397	}
398
399	if (cluster_info->uncore_data.agent_type_mask & AGENT_TYPE_CORE) {
400		cluster_info->uncore_data.domain_id = cluster_info->cdie_id;
401		return;
402	}
403
404	/* Unlikely but cdie_mask may have holes, so take range */
405	cdie_range = fls(plat_info->cdie_mask) - ffs(plat_info->cdie_mask) + 1;
406	max_dies = topology_max_dies_per_package();
407
408	/*
409	 * If the CPU doesn't enumerate dies, then use current cdie range
410	 * as the max.
411	 */
412	if (cdie_range > max_dies)
413		max_dies = cdie_range;
414
415	guard(mutex)(&domain_lock);
416
417	if (!io_die_index_next)
418		io_die_index_next = max_dies;
419
420	if (!io_die_start[plat_info->partition]) {
421		io_die_start[plat_info->partition] = io_die_index_next;
422		/*
423		 * number of IO dies = num_resources - cdie_range. Hence
424		 * next partition io_die_index_next is set after IO dies
425		 * in the current partition.
426		 */
427		io_die_index_next += (num_resources - cdie_range);
428	}
429
430	/*
431	 * Index from IO die start within the partition:
432	 * This is the first valid domain after the cdies.
433	 * For example the current resource index 5 and cdies end at
434	 * index 3 (cdie_cnt = 4). Then the IO only index 5 - 4 = 1.
435	 */
436	part_io_index = id - cdie_range;
437
438	/*
439	 * Add to the IO die start index for this partition in this package
440	 * to make unique in the package.
441	 */
442	pkg_io_index = io_die_start[plat_info->partition] + part_io_index;
443
444	/* Assign this to domain ID */
445	cluster_info->uncore_data.domain_id = pkg_io_index;
446}
447
448/* Callback for sysfs read for TPMI uncore values. Called under mutex locks. */
449static int uncore_read(struct uncore_data *data, unsigned int *value, enum uncore_index index)
450{
451	struct tpmi_uncore_cluster_info *cluster_info;
452	int ret;
453
454	switch (index) {
455	case UNCORE_INDEX_MIN_FREQ:
456	case UNCORE_INDEX_MAX_FREQ:
457		return uncore_read_control_freq(data, value, index);
458
459	case UNCORE_INDEX_CURRENT_FREQ:
460		return uncore_read_freq(data, value);
461
462	case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD:
463	case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD:
464	case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE:
465	case UNCORE_INDEX_EFF_LAT_CTRL_FREQ:
466		return read_eff_lat_ctrl(data, value, index);
467
468	case UNCORE_INDEX_DIE_ID:
469		cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
470		ret = tpmi_get_linux_die_id(cluster_info->uncore_data.package_id,
471					    cluster_info->cdie_id);
472		if (ret < 0)
473			return ret;
474
475		*value = ret;
476		return 0;
477
478	default:
479		break;
480	}
481
482	return -EOPNOTSUPP;
483}
484
485/* Callback for sysfs write for TPMI uncore data. Called under mutex locks. */
486static int uncore_write(struct uncore_data *data, unsigned int value, enum uncore_index index)
487{
488	switch (index) {
489	case UNCORE_INDEX_EFF_LAT_CTRL_LOW_THRESHOLD:
490	case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD:
491	case UNCORE_INDEX_EFF_LAT_CTRL_HIGH_THRESHOLD_ENABLE:
492	case UNCORE_INDEX_EFF_LAT_CTRL_FREQ:
493		return write_eff_lat_ctrl(data, value, index);
494
495	case UNCORE_INDEX_MIN_FREQ:
496	case UNCORE_INDEX_MAX_FREQ:
497		return uncore_write_control_freq(data, value, index);
498
499	default:
500		break;
501	}
502
503	return -EOPNOTSUPP;
504}
505
506static void remove_cluster_entries(struct tpmi_uncore_struct *tpmi_uncore)
507{
508	int i;
509
510	for (i = 0; i < tpmi_uncore->power_domain_count; ++i) {
511		struct tpmi_uncore_power_domain_info *pd_info;
512		int j;
513
514		pd_info = &tpmi_uncore->pd_info[i];
515		if (!pd_info->uncore_base)
516			continue;
517
518		for (j = 0; j < pd_info->cluster_count; ++j) {
519			struct tpmi_uncore_cluster_info *cluster_info;
520
521			cluster_info = &pd_info->cluster_infos[j];
522			uncore_freq_remove_die_entry(&cluster_info->uncore_data);
523		}
524	}
525}
526
527static void set_cdie_id(int domain_id, struct tpmi_uncore_cluster_info *cluster_info,
528			struct oobmsm_plat_info *plat_info)
529{
530
531	cluster_info->cdie_id = domain_id;
532
533	if (plat_info->cdie_mask && cluster_info->uncore_data.agent_type_mask & AGENT_TYPE_CORE)
534		cluster_info->cdie_id = domain_id + ffs(plat_info->cdie_mask) - 1;
535}
536
537#define UNCORE_VERSION_MASK			GENMASK_ULL(7, 0)
538#define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK	GENMASK_ULL(15, 8)
539#define UNCORE_CLUSTER_OFF_MASK			GENMASK_ULL(7, 0)
540#define UNCORE_AUTONOMOUS_UFS_DISABLED		BIT(32)
541#define UNCORE_MAX_CLUSTER_PER_DOMAIN		8
542
543static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id)
544{
545	bool read_blocked = 0, write_blocked = 0;
546	struct oobmsm_plat_info *plat_info;
547	struct tpmi_uncore_struct *tpmi_uncore;
548	bool uncore_sysfs_added = false;
549	int ret, i, pkg = 0;
550	int num_resources;
551
552	ret = tpmi_get_feature_status(auxdev, TPMI_ID_UNCORE, &read_blocked, &write_blocked);
553	if (ret)
554		dev_info(&auxdev->dev, "Can't read feature status: ignoring blocked status\n");
555
556	if (read_blocked) {
557		dev_info(&auxdev->dev, "Firmware has blocked reads, exiting\n");
558		return -ENODEV;
559	}
560
561	/* Get number of power domains, which is equal to number of resources */
562	num_resources = tpmi_get_resource_count(auxdev);
563	if (!num_resources)
564		return -EINVAL;
565
566	/* Register callbacks to uncore core */
567	ret = uncore_freq_common_init(uncore_read, uncore_write);
568	if (ret)
569		return ret;
570
571	/* Allocate uncore instance per package */
572	tpmi_uncore = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_uncore), GFP_KERNEL);
573	if (!tpmi_uncore) {
574		ret = -ENOMEM;
575		goto err_rem_common;
576	}
577
578	/* Allocate memory for all power domains in a package */
579	tpmi_uncore->pd_info = devm_kcalloc(&auxdev->dev, num_resources,
580					    sizeof(*tpmi_uncore->pd_info),
581					    GFP_KERNEL);
582	if (!tpmi_uncore->pd_info) {
583		ret = -ENOMEM;
584		goto err_rem_common;
585	}
586
587	tpmi_uncore->power_domain_count = num_resources;
588	tpmi_uncore->write_blocked = write_blocked;
589
590	/* Get the package ID from the TPMI core */
591	plat_info = tpmi_get_platform_data(auxdev);
592	if (unlikely(!plat_info)) {
593		dev_info(&auxdev->dev, "Platform information is NULL\n");
594		ret = -ENODEV;
595		goto err_rem_common;
596	}
597
598	pkg = plat_info->package_id;
599
600	for (i = 0; i < num_resources; ++i) {
601		struct tpmi_uncore_power_domain_info *pd_info;
602		bool auto_ufs_enabled;
603		struct resource *res;
604		u64 cluster_offset;
605		u8 cluster_mask;
606		int mask, j;
607		u64 header;
608
609		res = tpmi_get_resource_at_index(auxdev, i);
610		if (!res)
611			continue;
612
613		pd_info = &tpmi_uncore->pd_info[i];
614
615		pd_info->uncore_base = devm_ioremap_resource(&auxdev->dev, res);
616		if (IS_ERR(pd_info->uncore_base)) {
617			ret = PTR_ERR(pd_info->uncore_base);
618			/*
619			 * Set to NULL so that clean up can still remove other
620			 * entries already created if any by
621			 * remove_cluster_entries()
622			 */
623			pd_info->uncore_base = NULL;
624			goto remove_clusters;
625		}
626
627		/* Check for version and skip this resource if there is mismatch */
628		header = readq(pd_info->uncore_base);
629		pd_info->ufs_header_ver = header & UNCORE_VERSION_MASK;
630
631		if (pd_info->ufs_header_ver == TPMI_VERSION_INVALID)
632			continue;
633
634		if (TPMI_MAJOR_VERSION(pd_info->ufs_header_ver) != UNCORE_MAJOR_VERSION) {
635			dev_err(&auxdev->dev, "Uncore: Unsupported major version:%lx\n",
636				TPMI_MAJOR_VERSION(pd_info->ufs_header_ver));
637			ret = -ENODEV;
638			goto remove_clusters;
639		}
640
641		if (TPMI_MINOR_VERSION(pd_info->ufs_header_ver) > UNCORE_MINOR_VERSION)
642			dev_info(&auxdev->dev, "Uncore: Ignore: Unsupported minor version:%lx\n",
643				 TPMI_MINOR_VERSION(pd_info->ufs_header_ver));
644
645		/* Get Cluster ID Mask */
646		cluster_mask = FIELD_GET(UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK, header);
647		if (!cluster_mask) {
648			dev_info(&auxdev->dev, "Uncore: Invalid cluster mask:%x\n", cluster_mask);
649			continue;
650		}
651
652		auto_ufs_enabled = !(header & UNCORE_AUTONOMOUS_UFS_DISABLED);
653
654		/* Find out number of clusters in this resource */
655		pd_info->cluster_count = hweight8(cluster_mask);
656
657		pd_info->cluster_infos = devm_kcalloc(&auxdev->dev, pd_info->cluster_count,
658						      sizeof(struct tpmi_uncore_cluster_info),
659						      GFP_KERNEL);
660		if (!pd_info->cluster_infos) {
661			ret = -ENOMEM;
662			goto remove_clusters;
663		}
664		/*
665		 * Each byte in the register point to status and control
666		 * registers belonging to cluster id 0-8.
667		 */
668		cluster_offset = readq(pd_info->uncore_base +
669					UNCORE_FABRIC_CLUSTER_OFFSET);
670
671		for (j = 0; j < pd_info->cluster_count; ++j) {
672			struct tpmi_uncore_cluster_info *cluster_info;
673
674			/* Get the offset for this cluster */
675			mask = (cluster_offset & UNCORE_CLUSTER_OFF_MASK);
676			/* Offset in QWORD, so change to bytes */
677			mask <<= 3;
678
679			cluster_info = &pd_info->cluster_infos[j];
680
681			cluster_info->cluster_base = pd_info->uncore_base + mask;
682
683			uncore_set_agent_type(cluster_info);
684
685			cluster_info->uncore_data.package_id = pkg;
686			/* There are no dies like Cascade Lake */
687			cluster_info->uncore_data.die_id = 0;
688			cluster_info->uncore_data.cluster_id = j;
689
690			set_cdie_id(i, cluster_info, plat_info);
691
692			set_domain_id(i, num_resources, plat_info, cluster_info);
693
694			cluster_info->uncore_root = tpmi_uncore;
695
696			if ((TPMI_MINOR_VERSION(pd_info->ufs_header_ver) >=
697			     UNCORE_ELC_SUPPORTED_VERSION) &&
698			    auto_ufs_enabled)
699				cluster_info->elc_supported = true;
700
701			ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0);
702			if (ret) {
703				cluster_info->cluster_base = NULL;
704				goto remove_clusters;
705			}
706			/* Point to next cluster offset */
707			cluster_offset >>= UNCORE_MAX_CLUSTER_PER_DOMAIN;
708			uncore_sysfs_added = true;
709		}
710	}
711
712	if (!uncore_sysfs_added) {
713		ret = -ENODEV;
714		goto remove_clusters;
715	}
716
717	auxiliary_set_drvdata(auxdev, tpmi_uncore);
718
719	if (topology_max_dies_per_package() > 1 || plat_info->partition)
720		return 0;
721
722	tpmi_uncore->root_cluster.root_domain = true;
723	tpmi_uncore->root_cluster.uncore_root = tpmi_uncore;
724
725	tpmi_uncore->root_cluster.uncore_data.package_id = pkg;
726	tpmi_uncore->root_cluster.uncore_data.domain_id = UNCORE_DOMAIN_ID_INVALID;
727	ret = uncore_freq_add_entry(&tpmi_uncore->root_cluster.uncore_data, 0);
728	if (ret)
729		goto remove_clusters;
730
731	return 0;
732
733remove_clusters:
734	remove_cluster_entries(tpmi_uncore);
735err_rem_common:
736	uncore_freq_common_exit();
737
738	return ret;
739}
740
741static void uncore_remove(struct auxiliary_device *auxdev)
742{
743	struct tpmi_uncore_struct *tpmi_uncore = auxiliary_get_drvdata(auxdev);
744
745	if (tpmi_uncore->root_cluster.root_domain)
746		uncore_freq_remove_die_entry(&tpmi_uncore->root_cluster.uncore_data);
747
748	remove_cluster_entries(tpmi_uncore);
749
750	uncore_freq_common_exit();
751}
752
753static const struct auxiliary_device_id intel_uncore_id_table[] = {
754	{ .name = "intel_vsec.tpmi-uncore" },
755	{}
756};
757MODULE_DEVICE_TABLE(auxiliary, intel_uncore_id_table);
758
759static struct auxiliary_driver intel_uncore_aux_driver = {
760	.id_table       = intel_uncore_id_table,
761	.remove         = uncore_remove,
762	.probe          = uncore_probe,
763};
764
765module_auxiliary_driver(intel_uncore_aux_driver);
766
767MODULE_IMPORT_NS("INTEL_TPMI");
768MODULE_IMPORT_NS("INTEL_UNCORE_FREQUENCY");
769MODULE_IMPORT_NS("INTEL_TPMI_POWER_DOMAIN");
770MODULE_DESCRIPTION("Intel TPMI UFS Driver");
771MODULE_LICENSE("GPL");