mpam_devices.c

   1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (C) 2025 Arm Ltd.
   3
   4#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__
   5
   6#include <linux/acpi.h>
   7#include <linux/atomic.h>
   8#include <linux/arm_mpam.h>
   9#include <linux/bitfield.h>
  10#include <linux/bitmap.h>
  11#include <linux/cacheinfo.h>
  12#include <linux/cpu.h>
  13#include <linux/cpumask.h>
  14#include <linux/device.h>
  15#include <linux/errno.h>
  16#include <linux/gfp.h>
  17#include <linux/interrupt.h>
  18#include <linux/irq.h>
  19#include <linux/irqdesc.h>
  20#include <linux/list.h>
  21#include <linux/lockdep.h>
  22#include <linux/mutex.h>
  23#include <linux/platform_device.h>
  24#include <linux/printk.h>
  25#include <linux/srcu.h>
  26#include <linux/spinlock.h>
  27#include <linux/types.h>
  28#include <linux/workqueue.h>
  29
  30#include "mpam_internal.h"
  31
  32DEFINE_STATIC_KEY_FALSE(mpam_enabled); /* This moves to arch code */
  33
  34/*
  35 * mpam_list_lock protects the SRCU lists when writing. Once the
  36 * mpam_enabled key is enabled these lists are read-only,
  37 * unless the error interrupt disables the driver.
  38 */
  39static DEFINE_MUTEX(mpam_list_lock);
  40static LIST_HEAD(mpam_all_msc);
  41
  42struct srcu_struct mpam_srcu;
  43
  44/*
  45 * Number of MSCs that have been probed. Once all MSCs have been probed MPAM
  46 * can be enabled.
  47 */
  48static atomic_t mpam_num_msc;
  49
  50static int mpam_cpuhp_state;
  51static DEFINE_MUTEX(mpam_cpuhp_state_lock);
  52
  53/*
  54 * The smallest common values for any CPU or MSC in the system.
  55 * Generating traffic outside this range will result in screaming interrupts.
  56 */
  57u16 mpam_partid_max;
  58u8 mpam_pmg_max;
  59static bool partid_max_init, partid_max_published;
  60static DEFINE_SPINLOCK(partid_max_lock);
  61
  62/*
  63 * mpam is enabled once all devices have been probed from CPU online callbacks,
  64 * scheduled via this work_struct. If access to an MSC depends on a CPU that
  65 * was not brought online at boot, this can happen surprisingly late.
  66 */
  67static DECLARE_WORK(mpam_enable_work, &mpam_enable);
  68
  69/*
  70 * All mpam error interrupts indicate a software bug. On receipt, disable the
  71 * driver.
  72 */
  73static DECLARE_WORK(mpam_broken_work, &mpam_disable);
  74
  75/* When mpam is disabled, the printed reason to aid debugging */
  76static char *mpam_disable_reason;
  77
  78/*
  79 * An MSC is a physical container for controls and monitors, each identified by
  80 * their RIS index. These share a base-address, interrupts and some MMIO
  81 * registers. A vMSC is a virtual container for RIS in an MSC that control or
  82 * monitor the same thing. Members of a vMSC are all RIS in the same MSC, but
  83 * not all RIS in an MSC share a vMSC.
  84 *
  85 * Components are a group of vMSC that control or monitor the same thing but
  86 * are from different MSC, so have different base-address, interrupts etc.
  87 * Classes are the set components of the same type.
  88 *
  89 * The features of a vMSC is the union of the RIS it contains.
  90 * The features of a Class and Component are the common subset of the vMSC
  91 * they contain.
  92 *
  93 * e.g. The system cache may have bandwidth controls on multiple interfaces,
  94 * for regulating traffic from devices independently of traffic from CPUs.
  95 * If these are two RIS in one MSC, they will be treated as controlling
  96 * different things, and will not share a vMSC/component/class.
  97 *
  98 * e.g. The L2 may have one MSC and two RIS, one for cache-controls another
  99 * for bandwidth. These two RIS are members of the same vMSC.
 100 *
 101 * e.g. The set of RIS that make up the L2 are grouped as a component. These
 102 * are sometimes termed slices. They should be configured the same, as if there
 103 * were only one.
 104 *
 105 * e.g. The SoC probably has more than one L2, each attached to a distinct set
 106 * of CPUs. All the L2 components are grouped as a class.
 107 *
 108 * When creating an MSC, struct mpam_msc is added to the all mpam_all_msc list,
 109 * then linked via struct mpam_ris to a vmsc, component and class.
 110 * The same MSC may exist under different class->component->vmsc paths, but the
 111 * RIS index will be unique.
 112 */
 113LIST_HEAD(mpam_classes);
 114
 115/* List of all objects that can be free()d after synchronise_srcu() */
 116static LLIST_HEAD(mpam_garbage);
 117
 118static inline void init_garbage(struct mpam_garbage *garbage)
 119{
 120	init_llist_node(&garbage->llist);
 121}
 122
 123#define add_to_garbage(x)				\
 124do {							\
 125	__typeof__(x) _x = (x);				\
 126	_x->garbage.to_free = _x;			\
 127	llist_add(&_x->garbage.llist, &mpam_garbage);	\
 128} while (0)
 129
 130static void mpam_free_garbage(void)
 131{
 132	struct mpam_garbage *iter, *tmp;
 133	struct llist_node *to_free = llist_del_all(&mpam_garbage);
 134
 135	if (!to_free)
 136		return;
 137
 138	synchronize_srcu(&mpam_srcu);
 139
 140	llist_for_each_entry_safe(iter, tmp, to_free, llist) {
 141		if (iter->pdev)
 142			devm_kfree(&iter->pdev->dev, iter->to_free);
 143		else
 144			kfree(iter->to_free);
 145	}
 146}
 147
 148/*
 149 * Once mpam is enabled, new requestors cannot further reduce the available
 150 * partid. Assert that the size is fixed, and new requestors will be turned
 151 * away.
 152 */
 153static void mpam_assert_partid_sizes_fixed(void)
 154{
 155	WARN_ON_ONCE(!partid_max_published);
 156}
 157
 158static u32 __mpam_read_reg(struct mpam_msc *msc, u16 reg)
 159{
 160	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
 161
 162	return readl_relaxed(msc->mapped_hwpage + reg);
 163}
 164
 165static inline u32 _mpam_read_partsel_reg(struct mpam_msc *msc, u16 reg)
 166{
 167	lockdep_assert_held_once(&msc->part_sel_lock);
 168	return __mpam_read_reg(msc, reg);
 169}
 170
 171#define mpam_read_partsel_reg(msc, reg) _mpam_read_partsel_reg(msc, MPAMF_##reg)
 172
 173static void __mpam_write_reg(struct mpam_msc *msc, u16 reg, u32 val)
 174{
 175	WARN_ON_ONCE(reg + sizeof(u32) > msc->mapped_hwpage_sz);
 176	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
 177
 178	writel_relaxed(val, msc->mapped_hwpage + reg);
 179}
 180
 181static inline void _mpam_write_partsel_reg(struct mpam_msc *msc, u16 reg, u32 val)
 182{
 183	lockdep_assert_held_once(&msc->part_sel_lock);
 184	__mpam_write_reg(msc, reg, val);
 185}
 186
 187#define mpam_write_partsel_reg(msc, reg, val)  _mpam_write_partsel_reg(msc, MPAMCFG_##reg, val)
 188
 189static inline u32 _mpam_read_monsel_reg(struct mpam_msc *msc, u16 reg)
 190{
 191	mpam_mon_sel_lock_held(msc);
 192	return __mpam_read_reg(msc, reg);
 193}
 194
 195#define mpam_read_monsel_reg(msc, reg) _mpam_read_monsel_reg(msc, MSMON_##reg)
 196
 197static inline void _mpam_write_monsel_reg(struct mpam_msc *msc, u16 reg, u32 val)
 198{
 199	mpam_mon_sel_lock_held(msc);
 200	__mpam_write_reg(msc, reg, val);
 201}
 202
 203#define mpam_write_monsel_reg(msc, reg, val)   _mpam_write_monsel_reg(msc, MSMON_##reg, val)
 204
 205static u64 mpam_msc_read_idr(struct mpam_msc *msc)
 206{
 207	u64 idr_high = 0, idr_low;
 208
 209	lockdep_assert_held(&msc->part_sel_lock);
 210
 211	idr_low = mpam_read_partsel_reg(msc, IDR);
 212	if (FIELD_GET(MPAMF_IDR_EXT, idr_low))
 213		idr_high = mpam_read_partsel_reg(msc, IDR + 4);
 214
 215	return (idr_high << 32) | idr_low;
 216}
 217
 218static void mpam_msc_clear_esr(struct mpam_msc *msc)
 219{
 220	u64 esr_low = __mpam_read_reg(msc, MPAMF_ESR);
 221
 222	if (!esr_low)
 223		return;
 224
 225	/*
 226	 * Clearing the high/low bits of MPAMF_ESR can not be atomic.
 227	 * Clear the top half first, so that the pending error bits in the
 228	 * lower half prevent hardware from updating either half of the
 229	 * register.
 230	 */
 231	if (msc->has_extd_esr)
 232		__mpam_write_reg(msc, MPAMF_ESR + 4, 0);
 233	__mpam_write_reg(msc, MPAMF_ESR, 0);
 234}
 235
 236static u64 mpam_msc_read_esr(struct mpam_msc *msc)
 237{
 238	u64 esr_high = 0, esr_low;
 239
 240	esr_low = __mpam_read_reg(msc, MPAMF_ESR);
 241	if (msc->has_extd_esr)
 242		esr_high = __mpam_read_reg(msc, MPAMF_ESR + 4);
 243
 244	return (esr_high << 32) | esr_low;
 245}
 246
 247static void __mpam_part_sel_raw(u32 partsel, struct mpam_msc *msc)
 248{
 249	lockdep_assert_held(&msc->part_sel_lock);
 250
 251	mpam_write_partsel_reg(msc, PART_SEL, partsel);
 252}
 253
 254static void __mpam_part_sel(u8 ris_idx, u16 partid, struct mpam_msc *msc)
 255{
 256	u32 partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) |
 257		      FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, partid);
 258
 259	__mpam_part_sel_raw(partsel, msc);
 260}
 261
 262static void __mpam_intpart_sel(u8 ris_idx, u16 intpartid, struct mpam_msc *msc)
 263{
 264	u32 partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) |
 265		      FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, intpartid) |
 266		      MPAMCFG_PART_SEL_INTERNAL;
 267
 268	__mpam_part_sel_raw(partsel, msc);
 269}
 270
 271int mpam_register_requestor(u16 partid_max, u8 pmg_max)
 272{
 273	guard(spinlock)(&partid_max_lock);
 274	if (!partid_max_init) {
 275		mpam_partid_max = partid_max;
 276		mpam_pmg_max = pmg_max;
 277		partid_max_init = true;
 278	} else if (!partid_max_published) {
 279		mpam_partid_max = min(mpam_partid_max, partid_max);
 280		mpam_pmg_max = min(mpam_pmg_max, pmg_max);
 281	} else {
 282		/* New requestors can't lower the values */
 283		if (partid_max < mpam_partid_max || pmg_max < mpam_pmg_max)
 284			return -EBUSY;
 285	}
 286
 287	return 0;
 288}
 289EXPORT_SYMBOL(mpam_register_requestor);
 290
 291static struct mpam_class *
 292mpam_class_alloc(u8 level_idx, enum mpam_class_types type)
 293{
 294	struct mpam_class *class;
 295
 296	lockdep_assert_held(&mpam_list_lock);
 297
 298	class = kzalloc(sizeof(*class), GFP_KERNEL);
 299	if (!class)
 300		return ERR_PTR(-ENOMEM);
 301	init_garbage(&class->garbage);
 302
 303	INIT_LIST_HEAD_RCU(&class->components);
 304	/* Affinity is updated when ris are added */
 305	class->level = level_idx;
 306	class->type = type;
 307	INIT_LIST_HEAD_RCU(&class->classes_list);
 308	ida_init(&class->ida_csu_mon);
 309	ida_init(&class->ida_mbwu_mon);
 310
 311	list_add_rcu(&class->classes_list, &mpam_classes);
 312
 313	return class;
 314}
 315
 316static void mpam_class_destroy(struct mpam_class *class)
 317{
 318	lockdep_assert_held(&mpam_list_lock);
 319
 320	list_del_rcu(&class->classes_list);
 321	add_to_garbage(class);
 322}
 323
 324static struct mpam_class *
 325mpam_class_find(u8 level_idx, enum mpam_class_types type)
 326{
 327	struct mpam_class *class;
 328
 329	lockdep_assert_held(&mpam_list_lock);
 330
 331	list_for_each_entry(class, &mpam_classes, classes_list) {
 332		if (class->type == type && class->level == level_idx)
 333			return class;
 334	}
 335
 336	return mpam_class_alloc(level_idx, type);
 337}
 338
 339static struct mpam_component *
 340mpam_component_alloc(struct mpam_class *class, int id)
 341{
 342	struct mpam_component *comp;
 343
 344	lockdep_assert_held(&mpam_list_lock);
 345
 346	comp = kzalloc(sizeof(*comp), GFP_KERNEL);
 347	if (!comp)
 348		return ERR_PTR(-ENOMEM);
 349	init_garbage(&comp->garbage);
 350
 351	comp->comp_id = id;
 352	INIT_LIST_HEAD_RCU(&comp->vmsc);
 353	/* Affinity is updated when RIS are added */
 354	INIT_LIST_HEAD_RCU(&comp->class_list);
 355	comp->class = class;
 356
 357	list_add_rcu(&comp->class_list, &class->components);
 358
 359	return comp;
 360}
 361
 362static void __destroy_component_cfg(struct mpam_component *comp);
 363
 364static void mpam_component_destroy(struct mpam_component *comp)
 365{
 366	struct mpam_class *class = comp->class;
 367
 368	lockdep_assert_held(&mpam_list_lock);
 369
 370	__destroy_component_cfg(comp);
 371
 372	list_del_rcu(&comp->class_list);
 373	add_to_garbage(comp);
 374
 375	if (list_empty(&class->components))
 376		mpam_class_destroy(class);
 377}
 378
 379static struct mpam_component *
 380mpam_component_find(struct mpam_class *class, int id)
 381{
 382	struct mpam_component *comp;
 383
 384	lockdep_assert_held(&mpam_list_lock);
 385
 386	list_for_each_entry(comp, &class->components, class_list) {
 387		if (comp->comp_id == id)
 388			return comp;
 389	}
 390
 391	return mpam_component_alloc(class, id);
 392}
 393
 394static struct mpam_vmsc *
 395mpam_vmsc_alloc(struct mpam_component *comp, struct mpam_msc *msc)
 396{
 397	struct mpam_vmsc *vmsc;
 398
 399	lockdep_assert_held(&mpam_list_lock);
 400
 401	vmsc = kzalloc(sizeof(*vmsc), GFP_KERNEL);
 402	if (!vmsc)
 403		return ERR_PTR(-ENOMEM);
 404	init_garbage(&vmsc->garbage);
 405
 406	INIT_LIST_HEAD_RCU(&vmsc->ris);
 407	INIT_LIST_HEAD_RCU(&vmsc->comp_list);
 408	vmsc->comp = comp;
 409	vmsc->msc = msc;
 410
 411	list_add_rcu(&vmsc->comp_list, &comp->vmsc);
 412
 413	return vmsc;
 414}
 415
 416static void mpam_vmsc_destroy(struct mpam_vmsc *vmsc)
 417{
 418	struct mpam_component *comp = vmsc->comp;
 419
 420	lockdep_assert_held(&mpam_list_lock);
 421
 422	list_del_rcu(&vmsc->comp_list);
 423	add_to_garbage(vmsc);
 424
 425	if (list_empty(&comp->vmsc))
 426		mpam_component_destroy(comp);
 427}
 428
 429static struct mpam_vmsc *
 430mpam_vmsc_find(struct mpam_component *comp, struct mpam_msc *msc)
 431{
 432	struct mpam_vmsc *vmsc;
 433
 434	lockdep_assert_held(&mpam_list_lock);
 435
 436	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
 437		if (vmsc->msc->id == msc->id)
 438			return vmsc;
 439	}
 440
 441	return mpam_vmsc_alloc(comp, msc);
 442}
 443
 444/*
 445 * The cacheinfo structures are only populated when CPUs are online.
 446 * This helper walks the acpi tables to include offline CPUs too.
 447 */
 448int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level,
 449				   cpumask_t *affinity)
 450{
 451	return acpi_pptt_get_cpumask_from_cache_id(cache_id, affinity);
 452}
 453
 454/*
 455 * cpumask_of_node() only knows about online CPUs. This can't tell us whether
 456 * a class is represented on all possible CPUs.
 457 */
 458static void get_cpumask_from_node_id(u32 node_id, cpumask_t *affinity)
 459{
 460	int cpu;
 461
 462	for_each_possible_cpu(cpu) {
 463		if (node_id == cpu_to_node(cpu))
 464			cpumask_set_cpu(cpu, affinity);
 465	}
 466}
 467
 468static int mpam_ris_get_affinity(struct mpam_msc *msc, cpumask_t *affinity,
 469				 enum mpam_class_types type,
 470				 struct mpam_class *class,
 471				 struct mpam_component *comp)
 472{
 473	int err;
 474
 475	switch (type) {
 476	case MPAM_CLASS_CACHE:
 477		err = mpam_get_cpumask_from_cache_id(comp->comp_id, class->level,
 478						     affinity);
 479		if (err) {
 480			dev_warn_once(&msc->pdev->dev,
 481				      "Failed to determine CPU affinity\n");
 482			return err;
 483		}
 484
 485		if (cpumask_empty(affinity))
 486			dev_warn_once(&msc->pdev->dev, "no CPUs associated with cache node\n");
 487
 488		break;
 489	case MPAM_CLASS_MEMORY:
 490		get_cpumask_from_node_id(comp->comp_id, affinity);
 491		/* affinity may be empty for CPU-less memory nodes */
 492		break;
 493	case MPAM_CLASS_UNKNOWN:
 494		return 0;
 495	}
 496
 497	cpumask_and(affinity, affinity, &msc->accessibility);
 498
 499	return 0;
 500}
 501
 502static int mpam_ris_create_locked(struct mpam_msc *msc, u8 ris_idx,
 503				  enum mpam_class_types type, u8 class_id,
 504				  int component_id)
 505{
 506	int err;
 507	struct mpam_vmsc *vmsc;
 508	struct mpam_msc_ris *ris;
 509	struct mpam_class *class;
 510	struct mpam_component *comp;
 511	struct platform_device *pdev = msc->pdev;
 512
 513	lockdep_assert_held(&mpam_list_lock);
 514
 515	if (ris_idx > MPAM_MSC_MAX_NUM_RIS)
 516		return -EINVAL;
 517
 518	if (test_and_set_bit(ris_idx, &msc->ris_idxs))
 519		return -EBUSY;
 520
 521	ris = devm_kzalloc(&msc->pdev->dev, sizeof(*ris), GFP_KERNEL);
 522	if (!ris)
 523		return -ENOMEM;
 524	init_garbage(&ris->garbage);
 525	ris->garbage.pdev = pdev;
 526
 527	class = mpam_class_find(class_id, type);
 528	if (IS_ERR(class))
 529		return PTR_ERR(class);
 530
 531	comp = mpam_component_find(class, component_id);
 532	if (IS_ERR(comp)) {
 533		if (list_empty(&class->components))
 534			mpam_class_destroy(class);
 535		return PTR_ERR(comp);
 536	}
 537
 538	vmsc = mpam_vmsc_find(comp, msc);
 539	if (IS_ERR(vmsc)) {
 540		if (list_empty(&comp->vmsc))
 541			mpam_component_destroy(comp);
 542		return PTR_ERR(vmsc);
 543	}
 544
 545	err = mpam_ris_get_affinity(msc, &ris->affinity, type, class, comp);
 546	if (err) {
 547		if (list_empty(&vmsc->ris))
 548			mpam_vmsc_destroy(vmsc);
 549		return err;
 550	}
 551
 552	ris->ris_idx = ris_idx;
 553	INIT_LIST_HEAD_RCU(&ris->msc_list);
 554	INIT_LIST_HEAD_RCU(&ris->vmsc_list);
 555	ris->vmsc = vmsc;
 556
 557	cpumask_or(&comp->affinity, &comp->affinity, &ris->affinity);
 558	cpumask_or(&class->affinity, &class->affinity, &ris->affinity);
 559	list_add_rcu(&ris->vmsc_list, &vmsc->ris);
 560	list_add_rcu(&ris->msc_list, &msc->ris);
 561
 562	return 0;
 563}
 564
 565static void mpam_ris_destroy(struct mpam_msc_ris *ris)
 566{
 567	struct mpam_vmsc *vmsc = ris->vmsc;
 568	struct mpam_msc *msc = vmsc->msc;
 569	struct mpam_component *comp = vmsc->comp;
 570	struct mpam_class *class = comp->class;
 571
 572	lockdep_assert_held(&mpam_list_lock);
 573
 574	/*
 575	 * It is assumed affinities don't overlap. If they do the class becomes
 576	 * unusable immediately.
 577	 */
 578	cpumask_andnot(&class->affinity, &class->affinity, &ris->affinity);
 579	cpumask_andnot(&comp->affinity, &comp->affinity, &ris->affinity);
 580	clear_bit(ris->ris_idx, &msc->ris_idxs);
 581	list_del_rcu(&ris->msc_list);
 582	list_del_rcu(&ris->vmsc_list);
 583	add_to_garbage(ris);
 584
 585	if (list_empty(&vmsc->ris))
 586		mpam_vmsc_destroy(vmsc);
 587}
 588
 589int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx,
 590		    enum mpam_class_types type, u8 class_id, int component_id)
 591{
 592	int err;
 593
 594	mutex_lock(&mpam_list_lock);
 595	err = mpam_ris_create_locked(msc, ris_idx, type, class_id,
 596				     component_id);
 597	mutex_unlock(&mpam_list_lock);
 598	if (err)
 599		mpam_free_garbage();
 600
 601	return err;
 602}
 603
 604static struct mpam_msc_ris *mpam_get_or_create_ris(struct mpam_msc *msc,
 605						   u8 ris_idx)
 606{
 607	int err;
 608	struct mpam_msc_ris *ris;
 609
 610	lockdep_assert_held(&mpam_list_lock);
 611
 612	if (!test_bit(ris_idx, &msc->ris_idxs)) {
 613		err = mpam_ris_create_locked(msc, ris_idx, MPAM_CLASS_UNKNOWN,
 614					     0, 0);
 615		if (err)
 616			return ERR_PTR(err);
 617	}
 618
 619	list_for_each_entry(ris, &msc->ris, msc_list) {
 620		if (ris->ris_idx == ris_idx)
 621			return ris;
 622	}
 623
 624	return ERR_PTR(-ENOENT);
 625}
 626
 627/*
 628 * IHI009A.a has this nugget: "If a monitor does not support automatic behaviour
 629 * of NRDY, software can use this bit for any purpose" - so hardware might not
 630 * implement this - but it isn't RES0.
 631 *
 632 * Try and see what values stick in this bit. If we can write either value,
 633 * its probably not implemented by hardware.
 634 */
 635static bool _mpam_ris_hw_probe_hw_nrdy(struct mpam_msc_ris *ris, u32 mon_reg)
 636{
 637	u32 now;
 638	u64 mon_sel;
 639	bool can_set, can_clear;
 640	struct mpam_msc *msc = ris->vmsc->msc;
 641
 642	if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc)))
 643		return false;
 644
 645	mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, 0) |
 646		  FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
 647	_mpam_write_monsel_reg(msc, mon_reg, mon_sel);
 648
 649	_mpam_write_monsel_reg(msc, mon_reg, MSMON___NRDY);
 650	now = _mpam_read_monsel_reg(msc, mon_reg);
 651	can_set = now & MSMON___NRDY;
 652
 653	_mpam_write_monsel_reg(msc, mon_reg, 0);
 654	now = _mpam_read_monsel_reg(msc, mon_reg);
 655	can_clear = !(now & MSMON___NRDY);
 656	mpam_mon_sel_unlock(msc);
 657
 658	return (!can_set || !can_clear);
 659}
 660
 661#define mpam_ris_hw_probe_hw_nrdy(_ris, _mon_reg)			\
 662	_mpam_ris_hw_probe_hw_nrdy(_ris, MSMON_##_mon_reg)
 663
 664static void mpam_ris_hw_probe(struct mpam_msc_ris *ris)
 665{
 666	int err;
 667	struct mpam_msc *msc = ris->vmsc->msc;
 668	struct device *dev = &msc->pdev->dev;
 669	struct mpam_props *props = &ris->props;
 670	struct mpam_class *class = ris->vmsc->comp->class;
 671
 672	lockdep_assert_held(&msc->probe_lock);
 673	lockdep_assert_held(&msc->part_sel_lock);
 674
 675	/* Cache Capacity Partitioning */
 676	if (FIELD_GET(MPAMF_IDR_HAS_CCAP_PART, ris->idr)) {
 677		u32 ccap_features = mpam_read_partsel_reg(msc, CCAP_IDR);
 678
 679		props->cmax_wd = FIELD_GET(MPAMF_CCAP_IDR_CMAX_WD, ccap_features);
 680		if (props->cmax_wd &&
 681		    FIELD_GET(MPAMF_CCAP_IDR_HAS_CMAX_SOFTLIM, ccap_features))
 682			mpam_set_feature(mpam_feat_cmax_softlim, props);
 683
 684		if (props->cmax_wd &&
 685		    !FIELD_GET(MPAMF_CCAP_IDR_NO_CMAX, ccap_features))
 686			mpam_set_feature(mpam_feat_cmax_cmax, props);
 687
 688		if (props->cmax_wd &&
 689		    FIELD_GET(MPAMF_CCAP_IDR_HAS_CMIN, ccap_features))
 690			mpam_set_feature(mpam_feat_cmax_cmin, props);
 691
 692		props->cassoc_wd = FIELD_GET(MPAMF_CCAP_IDR_CASSOC_WD, ccap_features);
 693		if (props->cassoc_wd &&
 694		    FIELD_GET(MPAMF_CCAP_IDR_HAS_CASSOC, ccap_features))
 695			mpam_set_feature(mpam_feat_cmax_cassoc, props);
 696	}
 697
 698	/* Cache Portion partitioning */
 699	if (FIELD_GET(MPAMF_IDR_HAS_CPOR_PART, ris->idr)) {
 700		u32 cpor_features = mpam_read_partsel_reg(msc, CPOR_IDR);
 701
 702		props->cpbm_wd = FIELD_GET(MPAMF_CPOR_IDR_CPBM_WD, cpor_features);
 703		if (props->cpbm_wd)
 704			mpam_set_feature(mpam_feat_cpor_part, props);
 705	}
 706
 707	/* Memory bandwidth partitioning */
 708	if (FIELD_GET(MPAMF_IDR_HAS_MBW_PART, ris->idr)) {
 709		u32 mbw_features = mpam_read_partsel_reg(msc, MBW_IDR);
 710
 711		/* portion bitmap resolution */
 712		props->mbw_pbm_bits = FIELD_GET(MPAMF_MBW_IDR_BWPBM_WD, mbw_features);
 713		if (props->mbw_pbm_bits &&
 714		    FIELD_GET(MPAMF_MBW_IDR_HAS_PBM, mbw_features))
 715			mpam_set_feature(mpam_feat_mbw_part, props);
 716
 717		props->bwa_wd = FIELD_GET(MPAMF_MBW_IDR_BWA_WD, mbw_features);
 718		if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MAX, mbw_features))
 719			mpam_set_feature(mpam_feat_mbw_max, props);
 720
 721		if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MIN, mbw_features))
 722			mpam_set_feature(mpam_feat_mbw_min, props);
 723
 724		if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_PROP, mbw_features))
 725			mpam_set_feature(mpam_feat_mbw_prop, props);
 726	}
 727
 728	/* Priority partitioning */
 729	if (FIELD_GET(MPAMF_IDR_HAS_PRI_PART, ris->idr)) {
 730		u32 pri_features = mpam_read_partsel_reg(msc, PRI_IDR);
 731
 732		props->intpri_wd = FIELD_GET(MPAMF_PRI_IDR_INTPRI_WD, pri_features);
 733		if (props->intpri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_INTPRI, pri_features)) {
 734			mpam_set_feature(mpam_feat_intpri_part, props);
 735			if (FIELD_GET(MPAMF_PRI_IDR_INTPRI_0_IS_LOW, pri_features))
 736				mpam_set_feature(mpam_feat_intpri_part_0_low, props);
 737		}
 738
 739		props->dspri_wd = FIELD_GET(MPAMF_PRI_IDR_DSPRI_WD, pri_features);
 740		if (props->dspri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_DSPRI, pri_features)) {
 741			mpam_set_feature(mpam_feat_dspri_part, props);
 742			if (FIELD_GET(MPAMF_PRI_IDR_DSPRI_0_IS_LOW, pri_features))
 743				mpam_set_feature(mpam_feat_dspri_part_0_low, props);
 744		}
 745	}
 746
 747	/* Performance Monitoring */
 748	if (FIELD_GET(MPAMF_IDR_HAS_MSMON, ris->idr)) {
 749		u32 msmon_features = mpam_read_partsel_reg(msc, MSMON_IDR);
 750
 751		/*
 752		 * If the firmware max-nrdy-us property is missing, the
 753		 * CSU counters can't be used. Should we wait forever?
 754		 */
 755		err = device_property_read_u32(&msc->pdev->dev,
 756					       "arm,not-ready-us",
 757					       &msc->nrdy_usec);
 758
 759		if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_CSU, msmon_features)) {
 760			u32 csumonidr;
 761
 762			csumonidr = mpam_read_partsel_reg(msc, CSUMON_IDR);
 763			props->num_csu_mon = FIELD_GET(MPAMF_CSUMON_IDR_NUM_MON, csumonidr);
 764			if (props->num_csu_mon) {
 765				bool hw_managed;
 766
 767				mpam_set_feature(mpam_feat_msmon_csu, props);
 768
 769				if (FIELD_GET(MPAMF_CSUMON_IDR_HAS_XCL, csumonidr))
 770					mpam_set_feature(mpam_feat_msmon_csu_xcl, props);
 771
 772				/* Is NRDY hardware managed? */
 773				hw_managed = mpam_ris_hw_probe_hw_nrdy(ris, CSU);
 774				if (hw_managed)
 775					mpam_set_feature(mpam_feat_msmon_csu_hw_nrdy, props);
 776			}
 777
 778			/*
 779			 * Accept the missing firmware property if NRDY appears
 780			 * un-implemented.
 781			 */
 782			if (err && mpam_has_feature(mpam_feat_msmon_csu_hw_nrdy, props))
 783				dev_err_once(dev, "Counters are not usable because not-ready timeout was not provided by firmware.");
 784		}
 785		if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_MBWU, msmon_features)) {
 786			bool has_long, hw_managed;
 787			u32 mbwumon_idr = mpam_read_partsel_reg(msc, MBWUMON_IDR);
 788
 789			props->num_mbwu_mon = FIELD_GET(MPAMF_MBWUMON_IDR_NUM_MON, mbwumon_idr);
 790			if (props->num_mbwu_mon) {
 791				mpam_set_feature(mpam_feat_msmon_mbwu, props);
 792
 793				if (FIELD_GET(MPAMF_MBWUMON_IDR_HAS_RWBW, mbwumon_idr))
 794					mpam_set_feature(mpam_feat_msmon_mbwu_rwbw, props);
 795
 796				has_long = FIELD_GET(MPAMF_MBWUMON_IDR_HAS_LONG, mbwumon_idr);
 797				if (has_long) {
 798					if (FIELD_GET(MPAMF_MBWUMON_IDR_LWD, mbwumon_idr))
 799						mpam_set_feature(mpam_feat_msmon_mbwu_63counter, props);
 800					else
 801						mpam_set_feature(mpam_feat_msmon_mbwu_44counter, props);
 802				} else {
 803					mpam_set_feature(mpam_feat_msmon_mbwu_31counter, props);
 804				}
 805
 806				/* Is NRDY hardware managed? */
 807				hw_managed = mpam_ris_hw_probe_hw_nrdy(ris, MBWU);
 808				if (hw_managed)
 809					mpam_set_feature(mpam_feat_msmon_mbwu_hw_nrdy, props);
 810
 811				/*
 812				 * Don't warn about any missing firmware property for
 813				 * MBWU NRDY - it doesn't make any sense!
 814				 */
 815			}
 816		}
 817	}
 818
 819	/*
 820	 * RIS with PARTID narrowing don't have enough storage for one
 821	 * configuration per PARTID. If these are in a class we could use,
 822	 * reduce the supported partid_max to match the number of intpartid.
 823	 * If the class is unknown, just ignore it.
 824	 */
 825	if (FIELD_GET(MPAMF_IDR_HAS_PARTID_NRW, ris->idr) &&
 826	    class->type != MPAM_CLASS_UNKNOWN) {
 827		u32 nrwidr = mpam_read_partsel_reg(msc, PARTID_NRW_IDR);
 828		u16 partid_max = FIELD_GET(MPAMF_PARTID_NRW_IDR_INTPARTID_MAX, nrwidr);
 829
 830		mpam_set_feature(mpam_feat_partid_nrw, props);
 831		msc->partid_max = min(msc->partid_max, partid_max);
 832	}
 833}
 834
 835static int mpam_msc_hw_probe(struct mpam_msc *msc)
 836{
 837	u64 idr;
 838	u16 partid_max;
 839	u8 ris_idx, pmg_max;
 840	struct mpam_msc_ris *ris;
 841	struct device *dev = &msc->pdev->dev;
 842
 843	lockdep_assert_held(&msc->probe_lock);
 844
 845	idr = __mpam_read_reg(msc, MPAMF_AIDR);
 846	if ((idr & MPAMF_AIDR_ARCH_MAJOR_REV) != MPAM_ARCHITECTURE_V1) {
 847		dev_err_once(dev, "MSC does not match MPAM architecture v1.x\n");
 848		return -EIO;
 849	}
 850
 851	/* Grab an IDR value to find out how many RIS there are */
 852	mutex_lock(&msc->part_sel_lock);
 853	idr = mpam_msc_read_idr(msc);
 854	mutex_unlock(&msc->part_sel_lock);
 855
 856	msc->ris_max = FIELD_GET(MPAMF_IDR_RIS_MAX, idr);
 857
 858	/* Use these values so partid/pmg always starts with a valid value */
 859	msc->partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr);
 860	msc->pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr);
 861
 862	for (ris_idx = 0; ris_idx <= msc->ris_max; ris_idx++) {
 863		mutex_lock(&msc->part_sel_lock);
 864		__mpam_part_sel(ris_idx, 0, msc);
 865		idr = mpam_msc_read_idr(msc);
 866		mutex_unlock(&msc->part_sel_lock);
 867
 868		partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr);
 869		pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr);
 870		msc->partid_max = min(msc->partid_max, partid_max);
 871		msc->pmg_max = min(msc->pmg_max, pmg_max);
 872		msc->has_extd_esr = FIELD_GET(MPAMF_IDR_HAS_EXTD_ESR, idr);
 873
 874		mutex_lock(&mpam_list_lock);
 875		ris = mpam_get_or_create_ris(msc, ris_idx);
 876		mutex_unlock(&mpam_list_lock);
 877		if (IS_ERR(ris))
 878			return PTR_ERR(ris);
 879		ris->idr = idr;
 880
 881		mutex_lock(&msc->part_sel_lock);
 882		__mpam_part_sel(ris_idx, 0, msc);
 883		mpam_ris_hw_probe(ris);
 884		mutex_unlock(&msc->part_sel_lock);
 885	}
 886
 887	/* Clear any stale errors */
 888	mpam_msc_clear_esr(msc);
 889
 890	spin_lock(&partid_max_lock);
 891	mpam_partid_max = min(mpam_partid_max, msc->partid_max);
 892	mpam_pmg_max = min(mpam_pmg_max, msc->pmg_max);
 893	spin_unlock(&partid_max_lock);
 894
 895	msc->probed = true;
 896
 897	return 0;
 898}
 899
 900struct mon_read {
 901	struct mpam_msc_ris		*ris;
 902	struct mon_cfg			*ctx;
 903	enum mpam_device_features	type;
 904	u64				*val;
 905	int				err;
 906};
 907
 908static bool mpam_ris_has_mbwu_long_counter(struct mpam_msc_ris *ris)
 909{
 910	return (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, &ris->props) ||
 911		mpam_has_feature(mpam_feat_msmon_mbwu_44counter, &ris->props));
 912}
 913
 914static u64 mpam_msc_read_mbwu_l(struct mpam_msc *msc)
 915{
 916	int retry = 3;
 917	u32 mbwu_l_low;
 918	u64 mbwu_l_high1, mbwu_l_high2;
 919
 920	mpam_mon_sel_lock_held(msc);
 921
 922	WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz);
 923	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
 924
 925	mbwu_l_high2 = __mpam_read_reg(msc, MSMON_MBWU_L + 4);
 926	do {
 927		mbwu_l_high1 = mbwu_l_high2;
 928		mbwu_l_low = __mpam_read_reg(msc, MSMON_MBWU_L);
 929		mbwu_l_high2 = __mpam_read_reg(msc, MSMON_MBWU_L + 4);
 930
 931		retry--;
 932	} while (mbwu_l_high1 != mbwu_l_high2 && retry > 0);
 933
 934	if (mbwu_l_high1 == mbwu_l_high2)
 935		return (mbwu_l_high1 << 32) | mbwu_l_low;
 936
 937	pr_warn("Failed to read a stable value\n");
 938	return MSMON___L_NRDY;
 939}
 940
 941static void mpam_msc_zero_mbwu_l(struct mpam_msc *msc)
 942{
 943	mpam_mon_sel_lock_held(msc);
 944
 945	WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz);
 946	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
 947
 948	__mpam_write_reg(msc, MSMON_MBWU_L, 0);
 949	__mpam_write_reg(msc, MSMON_MBWU_L + 4, 0);
 950}
 951
 952static void gen_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val,
 953				   u32 *flt_val)
 954{
 955	struct mon_cfg *ctx = m->ctx;
 956
 957	/*
 958	 * For CSU counters its implementation-defined what happens when not
 959	 * filtering by partid.
 960	 */
 961	*ctl_val = MSMON_CFG_x_CTL_MATCH_PARTID;
 962
 963	*flt_val = FIELD_PREP(MSMON_CFG_x_FLT_PARTID, ctx->partid);
 964
 965	if (m->ctx->match_pmg) {
 966		*ctl_val |= MSMON_CFG_x_CTL_MATCH_PMG;
 967		*flt_val |= FIELD_PREP(MSMON_CFG_x_FLT_PMG, ctx->pmg);
 968	}
 969
 970	switch (m->type) {
 971	case mpam_feat_msmon_csu:
 972		*ctl_val |= MSMON_CFG_CSU_CTL_TYPE_CSU;
 973
 974		if (mpam_has_feature(mpam_feat_msmon_csu_xcl, &m->ris->props))
 975			*flt_val |= FIELD_PREP(MSMON_CFG_CSU_FLT_XCL, ctx->csu_exclude_clean);
 976
 977		break;
 978	case mpam_feat_msmon_mbwu_31counter:
 979	case mpam_feat_msmon_mbwu_44counter:
 980	case mpam_feat_msmon_mbwu_63counter:
 981		*ctl_val |= MSMON_CFG_MBWU_CTL_TYPE_MBWU;
 982
 983		if (mpam_has_feature(mpam_feat_msmon_mbwu_rwbw, &m->ris->props))
 984			*flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_RWBW, ctx->opts);
 985
 986		break;
 987	default:
 988		pr_warn("Unexpected monitor type %d\n", m->type);
 989	}
 990}
 991
 992static void read_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val,
 993				    u32 *flt_val)
 994{
 995	struct mpam_msc *msc = m->ris->vmsc->msc;
 996
 997	switch (m->type) {
 998	case mpam_feat_msmon_csu:
 999		*ctl_val = mpam_read_monsel_reg(msc, CFG_CSU_CTL);
1000		*flt_val = mpam_read_monsel_reg(msc, CFG_CSU_FLT);
1001		break;
1002	case mpam_feat_msmon_mbwu_31counter:
1003	case mpam_feat_msmon_mbwu_44counter:
1004	case mpam_feat_msmon_mbwu_63counter:
1005		*ctl_val = mpam_read_monsel_reg(msc, CFG_MBWU_CTL);
1006		*flt_val = mpam_read_monsel_reg(msc, CFG_MBWU_FLT);
1007		break;
1008	default:
1009		pr_warn("Unexpected monitor type %d\n", m->type);
1010	}
1011}
1012
1013/* Remove values set by the hardware to prevent apparent mismatches. */
1014static inline void clean_msmon_ctl_val(u32 *cur_ctl)
1015{
1016	*cur_ctl &= ~MSMON_CFG_x_CTL_OFLOW_STATUS;
1017
1018	if (FIELD_GET(MSMON_CFG_x_CTL_TYPE, *cur_ctl) == MSMON_CFG_MBWU_CTL_TYPE_MBWU)
1019		*cur_ctl &= ~MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L;
1020}
1021
1022static void write_msmon_ctl_flt_vals(struct mon_read *m, u32 ctl_val,
1023				     u32 flt_val)
1024{
1025	struct mpam_msc *msc = m->ris->vmsc->msc;
1026
1027	/*
1028	 * Write the ctl_val with the enable bit cleared, reset the counter,
1029	 * then enable counter.
1030	 */
1031	switch (m->type) {
1032	case mpam_feat_msmon_csu:
1033		mpam_write_monsel_reg(msc, CFG_CSU_FLT, flt_val);
1034		mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val);
1035		mpam_write_monsel_reg(msc, CSU, 0);
1036		mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val | MSMON_CFG_x_CTL_EN);
1037		break;
1038	case mpam_feat_msmon_mbwu_31counter:
1039	case mpam_feat_msmon_mbwu_44counter:
1040	case mpam_feat_msmon_mbwu_63counter:
1041		mpam_write_monsel_reg(msc, CFG_MBWU_FLT, flt_val);
1042		mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val);
1043		mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val | MSMON_CFG_x_CTL_EN);
1044		/* Counting monitors require NRDY to be reset by software */
1045		if (m->type == mpam_feat_msmon_mbwu_31counter)
1046			mpam_write_monsel_reg(msc, MBWU, 0);
1047		else
1048			mpam_msc_zero_mbwu_l(m->ris->vmsc->msc);
1049		break;
1050	default:
1051		pr_warn("Unexpected monitor type %d\n", m->type);
1052	}
1053}
1054
1055static u64 mpam_msmon_overflow_val(enum mpam_device_features type)
1056{
1057	/* TODO: implement scaling counters */
1058	switch (type) {
1059	case mpam_feat_msmon_mbwu_63counter:
1060		return BIT_ULL(hweight_long(MSMON___LWD_VALUE));
1061	case mpam_feat_msmon_mbwu_44counter:
1062		return BIT_ULL(hweight_long(MSMON___L_VALUE));
1063	case mpam_feat_msmon_mbwu_31counter:
1064		return BIT_ULL(hweight_long(MSMON___VALUE));
1065	default:
1066		return 0;
1067	}
1068}
1069
1070static void __ris_msmon_read(void *arg)
1071{
1072	u64 now;
1073	bool nrdy = false;
1074	bool config_mismatch;
1075	bool overflow = false;
1076	struct mon_read *m = arg;
1077	struct mon_cfg *ctx = m->ctx;
1078	bool reset_on_next_read = false;
1079	struct mpam_msc_ris *ris = m->ris;
1080	struct msmon_mbwu_state *mbwu_state;
1081	struct mpam_props *rprops = &ris->props;
1082	struct mpam_msc *msc = m->ris->vmsc->msc;
1083	u32 mon_sel, ctl_val, flt_val, cur_ctl, cur_flt;
1084
1085	if (!mpam_mon_sel_lock(msc)) {
1086		m->err = -EIO;
1087		return;
1088	}
1089	mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, ctx->mon) |
1090		  FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
1091	mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
1092
1093	switch (m->type) {
1094	case mpam_feat_msmon_mbwu_31counter:
1095	case mpam_feat_msmon_mbwu_44counter:
1096	case mpam_feat_msmon_mbwu_63counter:
1097		mbwu_state = &ris->mbwu_state[ctx->mon];
1098		if (mbwu_state) {
1099			reset_on_next_read = mbwu_state->reset_on_next_read;
1100			mbwu_state->reset_on_next_read = false;
1101		}
1102		break;
1103	default:
1104		break;
1105	}
1106
1107	/*
1108	 * Read the existing configuration to avoid re-writing the same values.
1109	 * This saves waiting for 'nrdy' on subsequent reads.
1110	 */
1111	read_msmon_ctl_flt_vals(m, &cur_ctl, &cur_flt);
1112
1113	if (mpam_feat_msmon_mbwu_31counter == m->type)
1114		overflow = cur_ctl & MSMON_CFG_x_CTL_OFLOW_STATUS;
1115	else if (mpam_feat_msmon_mbwu_44counter == m->type ||
1116		 mpam_feat_msmon_mbwu_63counter == m->type)
1117		overflow = cur_ctl & MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L;
1118
1119	clean_msmon_ctl_val(&cur_ctl);
1120	gen_msmon_ctl_flt_vals(m, &ctl_val, &flt_val);
1121	config_mismatch = cur_flt != flt_val ||
1122			  cur_ctl != (ctl_val | MSMON_CFG_x_CTL_EN);
1123
1124	if (config_mismatch || reset_on_next_read) {
1125		write_msmon_ctl_flt_vals(m, ctl_val, flt_val);
1126		overflow = false;
1127	} else if (overflow) {
1128		mpam_write_monsel_reg(msc, CFG_MBWU_CTL,
1129				      cur_ctl &
1130				      ~(MSMON_CFG_x_CTL_OFLOW_STATUS |
1131					MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L));
1132	}
1133
1134	switch (m->type) {
1135	case mpam_feat_msmon_csu:
1136		now = mpam_read_monsel_reg(msc, CSU);
1137		if (mpam_has_feature(mpam_feat_msmon_csu_hw_nrdy, rprops))
1138			nrdy = now & MSMON___NRDY;
1139		now = FIELD_GET(MSMON___VALUE, now);
1140		break;
1141	case mpam_feat_msmon_mbwu_31counter:
1142	case mpam_feat_msmon_mbwu_44counter:
1143	case mpam_feat_msmon_mbwu_63counter:
1144		if (m->type != mpam_feat_msmon_mbwu_31counter) {
1145			now = mpam_msc_read_mbwu_l(msc);
1146			if (mpam_has_feature(mpam_feat_msmon_mbwu_hw_nrdy, rprops))
1147				nrdy = now & MSMON___L_NRDY;
1148
1149			if (m->type == mpam_feat_msmon_mbwu_63counter)
1150				now = FIELD_GET(MSMON___LWD_VALUE, now);
1151			else
1152				now = FIELD_GET(MSMON___L_VALUE, now);
1153		} else {
1154			now = mpam_read_monsel_reg(msc, MBWU);
1155			if (mpam_has_feature(mpam_feat_msmon_mbwu_hw_nrdy, rprops))
1156				nrdy = now & MSMON___NRDY;
1157			now = FIELD_GET(MSMON___VALUE, now);
1158		}
1159
1160		if (nrdy)
1161			break;
1162
1163		mbwu_state = &ris->mbwu_state[ctx->mon];
1164
1165		if (overflow)
1166			mbwu_state->correction += mpam_msmon_overflow_val(m->type);
1167
1168		/*
1169		 * Include bandwidth consumed before the last hardware reset and
1170		 * a counter size increment for each overflow.
1171		 */
1172		now += mbwu_state->correction;
1173		break;
1174	default:
1175		m->err = -EINVAL;
1176	}
1177	mpam_mon_sel_unlock(msc);
1178
1179	if (nrdy)
1180		m->err = -EBUSY;
1181
1182	if (m->err)
1183		return;
1184
1185	*m->val += now;
1186}
1187
1188static int _msmon_read(struct mpam_component *comp, struct mon_read *arg)
1189{
1190	int err, any_err = 0;
1191	struct mpam_vmsc *vmsc;
1192
1193	guard(srcu)(&mpam_srcu);
1194	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
1195				 srcu_read_lock_held(&mpam_srcu)) {
1196		struct mpam_msc *msc = vmsc->msc;
1197		struct mpam_msc_ris *ris;
1198
1199		list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
1200					 srcu_read_lock_held(&mpam_srcu)) {
1201			arg->ris = ris;
1202
1203			err = smp_call_function_any(&msc->accessibility,
1204						    __ris_msmon_read, arg,
1205						    true);
1206			if (!err && arg->err)
1207				err = arg->err;
1208
1209			/*
1210			 * Save one error to be returned to the caller, but
1211			 * keep reading counters so that get reprogrammed. On
1212			 * platforms with NRDY this lets us wait once.
1213			 */
1214			if (err)
1215				any_err = err;
1216		}
1217	}
1218
1219	return any_err;
1220}
1221
1222static enum mpam_device_features mpam_msmon_choose_counter(struct mpam_class *class)
1223{
1224	struct mpam_props *cprops = &class->props;
1225
1226	if (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, cprops))
1227		return mpam_feat_msmon_mbwu_63counter;
1228	if (mpam_has_feature(mpam_feat_msmon_mbwu_44counter, cprops))
1229		return mpam_feat_msmon_mbwu_44counter;
1230
1231	return mpam_feat_msmon_mbwu_31counter;
1232}
1233
1234int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx,
1235		    enum mpam_device_features type, u64 *val)
1236{
1237	int err;
1238	struct mon_read arg;
1239	u64 wait_jiffies = 0;
1240	struct mpam_class *class = comp->class;
1241	struct mpam_props *cprops = &class->props;
1242
1243	might_sleep();
1244
1245	if (!mpam_is_enabled())
1246		return -EIO;
1247
1248	if (!mpam_has_feature(type, cprops))
1249		return -EOPNOTSUPP;
1250
1251	if (type == mpam_feat_msmon_mbwu)
1252		type = mpam_msmon_choose_counter(class);
1253
1254	arg = (struct mon_read) {
1255		.ctx = ctx,
1256		.type = type,
1257		.val = val,
1258	};
1259	*val = 0;
1260
1261	err = _msmon_read(comp, &arg);
1262	if (err == -EBUSY && class->nrdy_usec)
1263		wait_jiffies = usecs_to_jiffies(class->nrdy_usec);
1264
1265	while (wait_jiffies)
1266		wait_jiffies = schedule_timeout_uninterruptible(wait_jiffies);
1267
1268	if (err == -EBUSY) {
1269		arg = (struct mon_read) {
1270			.ctx = ctx,
1271			.type = type,
1272			.val = val,
1273		};
1274		*val = 0;
1275
1276		err = _msmon_read(comp, &arg);
1277	}
1278
1279	return err;
1280}
1281
1282void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx)
1283{
1284	struct mpam_msc *msc;
1285	struct mpam_vmsc *vmsc;
1286	struct mpam_msc_ris *ris;
1287
1288	if (!mpam_is_enabled())
1289		return;
1290
1291	guard(srcu)(&mpam_srcu);
1292	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
1293				 srcu_read_lock_held(&mpam_srcu)) {
1294		if (!mpam_has_feature(mpam_feat_msmon_mbwu, &vmsc->props))
1295			continue;
1296
1297		msc = vmsc->msc;
1298		list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
1299					 srcu_read_lock_held(&mpam_srcu)) {
1300			if (!mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props))
1301				continue;
1302
1303			if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc)))
1304				continue;
1305
1306			ris->mbwu_state[ctx->mon].correction = 0;
1307			ris->mbwu_state[ctx->mon].reset_on_next_read = true;
1308			mpam_mon_sel_unlock(msc);
1309		}
1310	}
1311}
1312
1313static void mpam_reset_msc_bitmap(struct mpam_msc *msc, u16 reg, u16 wd)
1314{
1315	u32 num_words, msb;
1316	u32 bm = ~0;
1317	int i;
1318
1319	lockdep_assert_held(&msc->part_sel_lock);
1320
1321	if (wd == 0)
1322		return;
1323
1324	/*
1325	 * Write all ~0 to all but the last 32bit-word, which may
1326	 * have fewer bits...
1327	 */
1328	num_words = DIV_ROUND_UP(wd, 32);
1329	for (i = 0; i < num_words - 1; i++, reg += sizeof(bm))
1330		__mpam_write_reg(msc, reg, bm);
1331
1332	/*
1333	 * ....and then the last (maybe) partial 32bit word. When wd is a
1334	 * multiple of 32, msb should be 31 to write a full 32bit word.
1335	 */
1336	msb = (wd - 1) % 32;
1337	bm = GENMASK(msb, 0);
1338	__mpam_write_reg(msc, reg, bm);
1339}
1340
1341/* Called via IPI. Call while holding an SRCU reference */
1342static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid,
1343				      struct mpam_config *cfg)
1344{
1345	u32 pri_val = 0;
1346	u16 cmax = MPAMCFG_CMAX_CMAX;
1347	struct mpam_msc *msc = ris->vmsc->msc;
1348	struct mpam_props *rprops = &ris->props;
1349	u16 dspri = GENMASK(rprops->dspri_wd, 0);
1350	u16 intpri = GENMASK(rprops->intpri_wd, 0);
1351
1352	mutex_lock(&msc->part_sel_lock);
1353	__mpam_part_sel(ris->ris_idx, partid, msc);
1354
1355	if (mpam_has_feature(mpam_feat_partid_nrw, rprops)) {
1356		/* Update the intpartid mapping */
1357		mpam_write_partsel_reg(msc, INTPARTID,
1358				       MPAMCFG_INTPARTID_INTERNAL | partid);
1359
1360		/*
1361		 * Then switch to the 'internal' partid to update the
1362		 * configuration.
1363		 */
1364		__mpam_intpart_sel(ris->ris_idx, partid, msc);
1365	}
1366
1367	if (mpam_has_feature(mpam_feat_cpor_part, rprops) &&
1368	    mpam_has_feature(mpam_feat_cpor_part, cfg)) {
1369		if (cfg->reset_cpbm)
1370			mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd);
1371		else
1372			mpam_write_partsel_reg(msc, CPBM, cfg->cpbm);
1373	}
1374
1375	if (mpam_has_feature(mpam_feat_mbw_part, rprops) &&
1376	    mpam_has_feature(mpam_feat_mbw_part, cfg)) {
1377		if (cfg->reset_mbw_pbm)
1378			mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits);
1379		else
1380			mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm);
1381	}
1382
1383	if (mpam_has_feature(mpam_feat_mbw_min, rprops) &&
1384	    mpam_has_feature(mpam_feat_mbw_min, cfg))
1385		mpam_write_partsel_reg(msc, MBW_MIN, 0);
1386
1387	if (mpam_has_feature(mpam_feat_mbw_max, rprops) &&
1388	    mpam_has_feature(mpam_feat_mbw_max, cfg)) {
1389		if (cfg->reset_mbw_max)
1390			mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX);
1391		else
1392			mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max);
1393	}
1394
1395	if (mpam_has_feature(mpam_feat_mbw_prop, rprops) &&
1396	    mpam_has_feature(mpam_feat_mbw_prop, cfg))
1397		mpam_write_partsel_reg(msc, MBW_PROP, 0);
1398
1399	if (mpam_has_feature(mpam_feat_cmax_cmax, rprops))
1400		mpam_write_partsel_reg(msc, CMAX, cmax);
1401
1402	if (mpam_has_feature(mpam_feat_cmax_cmin, rprops))
1403		mpam_write_partsel_reg(msc, CMIN, 0);
1404
1405	if (mpam_has_feature(mpam_feat_cmax_cassoc, rprops))
1406		mpam_write_partsel_reg(msc, CASSOC, MPAMCFG_CASSOC_CASSOC);
1407
1408	if (mpam_has_feature(mpam_feat_intpri_part, rprops) ||
1409	    mpam_has_feature(mpam_feat_dspri_part, rprops)) {
1410		/* aces high? */
1411		if (!mpam_has_feature(mpam_feat_intpri_part_0_low, rprops))
1412			intpri = 0;
1413		if (!mpam_has_feature(mpam_feat_dspri_part_0_low, rprops))
1414			dspri = 0;
1415
1416		if (mpam_has_feature(mpam_feat_intpri_part, rprops))
1417			pri_val |= FIELD_PREP(MPAMCFG_PRI_INTPRI, intpri);
1418		if (mpam_has_feature(mpam_feat_dspri_part, rprops))
1419			pri_val |= FIELD_PREP(MPAMCFG_PRI_DSPRI, dspri);
1420
1421		mpam_write_partsel_reg(msc, PRI, pri_val);
1422	}
1423
1424	mutex_unlock(&msc->part_sel_lock);
1425}
1426
1427/* Call with msc cfg_lock held */
1428static int mpam_restore_mbwu_state(void *_ris)
1429{
1430	int i;
1431	u64 val;
1432	struct mon_read mwbu_arg;
1433	struct mpam_msc_ris *ris = _ris;
1434	struct mpam_class *class = ris->vmsc->comp->class;
1435
1436	for (i = 0; i < ris->props.num_mbwu_mon; i++) {
1437		if (ris->mbwu_state[i].enabled) {
1438			mwbu_arg.ris = ris;
1439			mwbu_arg.ctx = &ris->mbwu_state[i].cfg;
1440			mwbu_arg.type = mpam_msmon_choose_counter(class);
1441			mwbu_arg.val = &val;
1442
1443			__ris_msmon_read(&mwbu_arg);
1444		}
1445	}
1446
1447	return 0;
1448}
1449
1450/* Call with MSC cfg_lock held */
1451static int mpam_save_mbwu_state(void *arg)
1452{
1453	int i;
1454	u64 val;
1455	struct mon_cfg *cfg;
1456	u32 cur_flt, cur_ctl, mon_sel;
1457	struct mpam_msc_ris *ris = arg;
1458	struct msmon_mbwu_state *mbwu_state;
1459	struct mpam_msc *msc = ris->vmsc->msc;
1460
1461	for (i = 0; i < ris->props.num_mbwu_mon; i++) {
1462		mbwu_state = &ris->mbwu_state[i];
1463		cfg = &mbwu_state->cfg;
1464
1465		if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc)))
1466			return -EIO;
1467
1468		mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, i) |
1469			  FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
1470		mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
1471
1472		cur_flt = mpam_read_monsel_reg(msc, CFG_MBWU_FLT);
1473		cur_ctl = mpam_read_monsel_reg(msc, CFG_MBWU_CTL);
1474		mpam_write_monsel_reg(msc, CFG_MBWU_CTL, 0);
1475
1476		if (mpam_ris_has_mbwu_long_counter(ris)) {
1477			val = mpam_msc_read_mbwu_l(msc);
1478			mpam_msc_zero_mbwu_l(msc);
1479		} else {
1480			val = mpam_read_monsel_reg(msc, MBWU);
1481			mpam_write_monsel_reg(msc, MBWU, 0);
1482		}
1483
1484		cfg->mon = i;
1485		cfg->pmg = FIELD_GET(MSMON_CFG_x_FLT_PMG, cur_flt);
1486		cfg->match_pmg = FIELD_GET(MSMON_CFG_x_CTL_MATCH_PMG, cur_ctl);
1487		cfg->partid = FIELD_GET(MSMON_CFG_x_FLT_PARTID, cur_flt);
1488		mbwu_state->correction += val;
1489		mbwu_state->enabled = FIELD_GET(MSMON_CFG_x_CTL_EN, cur_ctl);
1490		mpam_mon_sel_unlock(msc);
1491	}
1492
1493	return 0;
1494}
1495
1496static void mpam_init_reset_cfg(struct mpam_config *reset_cfg)
1497{
1498	*reset_cfg = (struct mpam_config) {
1499		.reset_cpbm = true,
1500		.reset_mbw_pbm = true,
1501		.reset_mbw_max = true,
1502	};
1503	bitmap_fill(reset_cfg->features, MPAM_FEATURE_LAST);
1504}
1505
1506/*
1507 * Called via smp_call_on_cpu() to prevent migration, while still being
1508 * pre-emptible. Caller must hold mpam_srcu.
1509 */
1510static int mpam_reset_ris(void *arg)
1511{
1512	u16 partid, partid_max;
1513	struct mpam_config reset_cfg;
1514	struct mpam_msc_ris *ris = arg;
1515
1516	if (ris->in_reset_state)
1517		return 0;
1518
1519	mpam_init_reset_cfg(&reset_cfg);
1520
1521	spin_lock(&partid_max_lock);
1522	partid_max = mpam_partid_max;
1523	spin_unlock(&partid_max_lock);
1524	for (partid = 0; partid <= partid_max; partid++)
1525		mpam_reprogram_ris_partid(ris, partid, &reset_cfg);
1526
1527	return 0;
1528}
1529
1530/*
1531 * Get the preferred CPU for this MSC. If it is accessible from this CPU,
1532 * this CPU is preferred. This can be preempted/migrated, it will only result
1533 * in more work.
1534 */
1535static int mpam_get_msc_preferred_cpu(struct mpam_msc *msc)
1536{
1537	int cpu = raw_smp_processor_id();
1538
1539	if (cpumask_test_cpu(cpu, &msc->accessibility))
1540		return cpu;
1541
1542	return cpumask_first_and(&msc->accessibility, cpu_online_mask);
1543}
1544
1545static int mpam_touch_msc(struct mpam_msc *msc, int (*fn)(void *a), void *arg)
1546{
1547	lockdep_assert_irqs_enabled();
1548	lockdep_assert_cpus_held();
1549	WARN_ON_ONCE(!srcu_read_lock_held((&mpam_srcu)));
1550
1551	return smp_call_on_cpu(mpam_get_msc_preferred_cpu(msc), fn, arg, true);
1552}
1553
1554struct mpam_write_config_arg {
1555	struct mpam_msc_ris *ris;
1556	struct mpam_component *comp;
1557	u16 partid;
1558};
1559
1560static int __write_config(void *arg)
1561{
1562	struct mpam_write_config_arg *c = arg;
1563
1564	mpam_reprogram_ris_partid(c->ris, c->partid, &c->comp->cfg[c->partid]);
1565
1566	return 0;
1567}
1568
1569static void mpam_reprogram_msc(struct mpam_msc *msc)
1570{
1571	u16 partid;
1572	bool reset;
1573	struct mpam_config *cfg;
1574	struct mpam_msc_ris *ris;
1575	struct mpam_write_config_arg arg;
1576
1577	/*
1578	 * No lock for mpam_partid_max as partid_max_published has been
1579	 * set by mpam_enabled(), so the values can no longer change.
1580	 */
1581	mpam_assert_partid_sizes_fixed();
1582
1583	mutex_lock(&msc->cfg_lock);
1584	list_for_each_entry_srcu(ris, &msc->ris, msc_list,
1585				 srcu_read_lock_held(&mpam_srcu)) {
1586		if (!mpam_is_enabled() && !ris->in_reset_state) {
1587			mpam_touch_msc(msc, &mpam_reset_ris, ris);
1588			ris->in_reset_state = true;
1589			continue;
1590		}
1591
1592		arg.comp = ris->vmsc->comp;
1593		arg.ris = ris;
1594		reset = true;
1595		for (partid = 0; partid <= mpam_partid_max; partid++) {
1596			cfg = &ris->vmsc->comp->cfg[partid];
1597			if (!bitmap_empty(cfg->features, MPAM_FEATURE_LAST))
1598				reset = false;
1599
1600			arg.partid = partid;
1601			mpam_touch_msc(msc, __write_config, &arg);
1602		}
1603		ris->in_reset_state = reset;
1604
1605		if (mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props))
1606			mpam_touch_msc(msc, &mpam_restore_mbwu_state, ris);
1607	}
1608	mutex_unlock(&msc->cfg_lock);
1609}
1610
1611static void _enable_percpu_irq(void *_irq)
1612{
1613	int *irq = _irq;
1614
1615	enable_percpu_irq(*irq, IRQ_TYPE_NONE);
1616}
1617
1618static int mpam_cpu_online(unsigned int cpu)
1619{
1620	struct mpam_msc *msc;
1621
1622	guard(srcu)(&mpam_srcu);
1623	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
1624				 srcu_read_lock_held(&mpam_srcu)) {
1625		if (!cpumask_test_cpu(cpu, &msc->accessibility))
1626			continue;
1627
1628		if (msc->reenable_error_ppi)
1629			_enable_percpu_irq(&msc->reenable_error_ppi);
1630
1631		if (atomic_fetch_inc(&msc->online_refs) == 0)
1632			mpam_reprogram_msc(msc);
1633	}
1634
1635	return 0;
1636}
1637
1638/* Before mpam is enabled, try to probe new MSC */
1639static int mpam_discovery_cpu_online(unsigned int cpu)
1640{
1641	int err = 0;
1642	struct mpam_msc *msc;
1643	bool new_device_probed = false;
1644
1645	if (mpam_is_enabled())
1646		return 0;
1647
1648	guard(srcu)(&mpam_srcu);
1649	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
1650				 srcu_read_lock_held(&mpam_srcu)) {
1651		if (!cpumask_test_cpu(cpu, &msc->accessibility))
1652			continue;
1653
1654		mutex_lock(&msc->probe_lock);
1655		if (!msc->probed)
1656			err = mpam_msc_hw_probe(msc);
1657		mutex_unlock(&msc->probe_lock);
1658
1659		if (err)
1660			break;
1661		new_device_probed = true;
1662	}
1663
1664	if (new_device_probed && !err)
1665		schedule_work(&mpam_enable_work);
1666	if (err) {
1667		mpam_disable_reason = "error during probing";
1668		schedule_work(&mpam_broken_work);
1669	}
1670
1671	return err;
1672}
1673
1674static int mpam_cpu_offline(unsigned int cpu)
1675{
1676	struct mpam_msc *msc;
1677
1678	guard(srcu)(&mpam_srcu);
1679	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
1680				 srcu_read_lock_held(&mpam_srcu)) {
1681		if (!cpumask_test_cpu(cpu, &msc->accessibility))
1682			continue;
1683
1684		if (msc->reenable_error_ppi)
1685			disable_percpu_irq(msc->reenable_error_ppi);
1686
1687		if (atomic_dec_and_test(&msc->online_refs)) {
1688			struct mpam_msc_ris *ris;
1689
1690			mutex_lock(&msc->cfg_lock);
1691			list_for_each_entry_srcu(ris, &msc->ris, msc_list,
1692						 srcu_read_lock_held(&mpam_srcu)) {
1693				mpam_touch_msc(msc, &mpam_reset_ris, ris);
1694
1695				/*
1696				 * The reset state for non-zero partid may be
1697				 * lost while the CPUs are offline.
1698				 */
1699				ris->in_reset_state = false;
1700
1701				if (mpam_is_enabled())
1702					mpam_touch_msc(msc, &mpam_save_mbwu_state, ris);
1703			}
1704			mutex_unlock(&msc->cfg_lock);
1705		}
1706	}
1707
1708	return 0;
1709}
1710
1711static void mpam_register_cpuhp_callbacks(int (*online)(unsigned int online),
1712					  int (*offline)(unsigned int offline),
1713					  char *name)
1714{
1715	mutex_lock(&mpam_cpuhp_state_lock);
1716	if (mpam_cpuhp_state) {
1717		cpuhp_remove_state(mpam_cpuhp_state);
1718		mpam_cpuhp_state = 0;
1719	}
1720
1721	mpam_cpuhp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, name, online,
1722					     offline);
1723	if (mpam_cpuhp_state <= 0) {
1724		pr_err("Failed to register cpuhp callbacks");
1725		mpam_cpuhp_state = 0;
1726	}
1727	mutex_unlock(&mpam_cpuhp_state_lock);
1728}
1729
1730static int __setup_ppi(struct mpam_msc *msc)
1731{
1732	int cpu;
1733
1734	msc->error_dev_id = alloc_percpu(struct mpam_msc *);
1735	if (!msc->error_dev_id)
1736		return -ENOMEM;
1737
1738	for_each_cpu(cpu, &msc->accessibility)
1739		*per_cpu_ptr(msc->error_dev_id, cpu) = msc;
1740
1741	return 0;
1742}
1743
1744static int mpam_msc_setup_error_irq(struct mpam_msc *msc)
1745{
1746	int irq;
1747
1748	irq = platform_get_irq_byname_optional(msc->pdev, "error");
1749	if (irq <= 0)
1750		return 0;
1751
1752	/* Allocate and initialise the percpu device pointer for PPI */
1753	if (irq_is_percpu(irq))
1754		return __setup_ppi(msc);
1755
1756	/* sanity check: shared interrupts can be routed anywhere? */
1757	if (!cpumask_equal(&msc->accessibility, cpu_possible_mask)) {
1758		pr_err_once("msc:%u is a private resource with a shared error interrupt",
1759			    msc->id);
1760		return -EINVAL;
1761	}
1762
1763	return 0;
1764}
1765
1766/*
1767 * An MSC can control traffic from a set of CPUs, but may only be accessible
1768 * from a (hopefully wider) set of CPUs. The common reason for this is power
1769 * management. If all the CPUs in a cluster are in PSCI:CPU_SUSPEND, the
1770 * corresponding cache may also be powered off. By making accesses from
1771 * one of those CPUs, we ensure we don't access a cache that's powered off.
1772 */
1773static void update_msc_accessibility(struct mpam_msc *msc)
1774{
1775	u32 affinity_id;
1776	int err;
1777
1778	err = device_property_read_u32(&msc->pdev->dev, "cpu_affinity",
1779				       &affinity_id);
1780	if (err)
1781		cpumask_copy(&msc->accessibility, cpu_possible_mask);
1782	else
1783		acpi_pptt_get_cpus_from_container(affinity_id, &msc->accessibility);
1784}
1785
1786/*
1787 * There are two ways of reaching a struct mpam_msc_ris. Via the
1788 * class->component->vmsc->ris, or via the msc.
1789 * When destroying the msc, the other side needs unlinking and cleaning up too.
1790 */
1791static void mpam_msc_destroy(struct mpam_msc *msc)
1792{
1793	struct platform_device *pdev = msc->pdev;
1794	struct mpam_msc_ris *ris, *tmp;
1795
1796	lockdep_assert_held(&mpam_list_lock);
1797
1798	list_for_each_entry_safe(ris, tmp, &msc->ris, msc_list)
1799		mpam_ris_destroy(ris);
1800
1801	list_del_rcu(&msc->all_msc_list);
1802	platform_set_drvdata(pdev, NULL);
1803
1804	add_to_garbage(msc);
1805}
1806
1807static void mpam_msc_drv_remove(struct platform_device *pdev)
1808{
1809	struct mpam_msc *msc = platform_get_drvdata(pdev);
1810
1811	mutex_lock(&mpam_list_lock);
1812	mpam_msc_destroy(msc);
1813	mutex_unlock(&mpam_list_lock);
1814
1815	mpam_free_garbage();
1816}
1817
1818static struct mpam_msc *do_mpam_msc_drv_probe(struct platform_device *pdev)
1819{
1820	int err;
1821	u32 tmp;
1822	struct mpam_msc *msc;
1823	struct resource *msc_res;
1824	struct device *dev = &pdev->dev;
1825
1826	lockdep_assert_held(&mpam_list_lock);
1827
1828	msc = devm_kzalloc(&pdev->dev, sizeof(*msc), GFP_KERNEL);
1829	if (!msc)
1830		return ERR_PTR(-ENOMEM);
1831	init_garbage(&msc->garbage);
1832	msc->garbage.pdev = pdev;
1833
1834	err = devm_mutex_init(dev, &msc->probe_lock);
1835	if (err)
1836		return ERR_PTR(err);
1837
1838	err = devm_mutex_init(dev, &msc->part_sel_lock);
1839	if (err)
1840		return ERR_PTR(err);
1841
1842	err = devm_mutex_init(dev, &msc->error_irq_lock);
1843	if (err)
1844		return ERR_PTR(err);
1845
1846	err = devm_mutex_init(dev, &msc->cfg_lock);
1847	if (err)
1848		return ERR_PTR(err);
1849
1850	mpam_mon_sel_lock_init(msc);
1851	msc->id = pdev->id;
1852	msc->pdev = pdev;
1853	INIT_LIST_HEAD_RCU(&msc->all_msc_list);
1854	INIT_LIST_HEAD_RCU(&msc->ris);
1855
1856	update_msc_accessibility(msc);
1857	if (cpumask_empty(&msc->accessibility)) {
1858		dev_err_once(dev, "MSC is not accessible from any CPU!");
1859		return ERR_PTR(-EINVAL);
1860	}
1861
1862	err = mpam_msc_setup_error_irq(msc);
1863	if (err)
1864		return ERR_PTR(err);
1865
1866	if (device_property_read_u32(&pdev->dev, "pcc-channel", &tmp))
1867		msc->iface = MPAM_IFACE_MMIO;
1868	else
1869		msc->iface = MPAM_IFACE_PCC;
1870
1871	if (msc->iface == MPAM_IFACE_MMIO) {
1872		void __iomem *io;
1873
1874		io = devm_platform_get_and_ioremap_resource(pdev, 0,
1875							    &msc_res);
1876		if (IS_ERR(io)) {
1877			dev_err_once(dev, "Failed to map MSC base address\n");
1878			return ERR_CAST(io);
1879		}
1880		msc->mapped_hwpage_sz = msc_res->end - msc_res->start;
1881		msc->mapped_hwpage = io;
1882	} else {
1883		return ERR_PTR(-EINVAL);
1884	}
1885
1886	list_add_rcu(&msc->all_msc_list, &mpam_all_msc);
1887	platform_set_drvdata(pdev, msc);
1888
1889	return msc;
1890}
1891
1892static int fw_num_msc;
1893
1894static int mpam_msc_drv_probe(struct platform_device *pdev)
1895{
1896	int err;
1897	struct mpam_msc *msc = NULL;
1898	void *plat_data = pdev->dev.platform_data;
1899
1900	mutex_lock(&mpam_list_lock);
1901	msc = do_mpam_msc_drv_probe(pdev);
1902	mutex_unlock(&mpam_list_lock);
1903
1904	if (IS_ERR(msc))
1905		return PTR_ERR(msc);
1906
1907	/* Create RIS entries described by firmware */
1908	err = acpi_mpam_parse_resources(msc, plat_data);
1909	if (err) {
1910		mpam_msc_drv_remove(pdev);
1911		return err;
1912	}
1913
1914	if (atomic_add_return(1, &mpam_num_msc) == fw_num_msc)
1915		mpam_register_cpuhp_callbacks(mpam_discovery_cpu_online, NULL,
1916					      "mpam:drv_probe");
1917
1918	return 0;
1919}
1920
1921static struct platform_driver mpam_msc_driver = {
1922	.driver = {
1923		.name = "mpam_msc",
1924	},
1925	.probe = mpam_msc_drv_probe,
1926	.remove = mpam_msc_drv_remove,
1927};
1928
1929/* Any of these features mean the BWA_WD field is valid. */
1930static bool mpam_has_bwa_wd_feature(struct mpam_props *props)
1931{
1932	if (mpam_has_feature(mpam_feat_mbw_min, props))
1933		return true;
1934	if (mpam_has_feature(mpam_feat_mbw_max, props))
1935		return true;
1936	if (mpam_has_feature(mpam_feat_mbw_prop, props))
1937		return true;
1938	return false;
1939}
1940
1941/* Any of these features mean the CMAX_WD field is valid. */
1942static bool mpam_has_cmax_wd_feature(struct mpam_props *props)
1943{
1944	if (mpam_has_feature(mpam_feat_cmax_cmax, props))
1945		return true;
1946	if (mpam_has_feature(mpam_feat_cmax_cmin, props))
1947		return true;
1948	return false;
1949}
1950
1951#define MISMATCHED_HELPER(parent, child, helper, field, alias)		\
1952	helper(parent) &&						\
1953	((helper(child) && (parent)->field != (child)->field) ||	\
1954	 (!helper(child) && !(alias)))
1955
1956#define MISMATCHED_FEAT(parent, child, feat, field, alias)		     \
1957	mpam_has_feature((feat), (parent)) &&				     \
1958	((mpam_has_feature((feat), (child)) && (parent)->field != (child)->field) || \
1959	 (!mpam_has_feature((feat), (child)) && !(alias)))
1960
1961#define CAN_MERGE_FEAT(parent, child, feat, alias)			\
1962	(alias) && !mpam_has_feature((feat), (parent)) &&		\
1963	mpam_has_feature((feat), (child))
1964
1965/*
1966 * Combine two props fields.
1967 * If this is for controls that alias the same resource, it is safe to just
1968 * copy the values over. If two aliasing controls implement the same scheme
1969 * a safe value must be picked.
1970 * For non-aliasing controls, these control different resources, and the
1971 * resulting safe value must be compatible with both. When merging values in
1972 * the tree, all the aliasing resources must be handled first.
1973 * On mismatch, parent is modified.
1974 */
1975static void __props_mismatch(struct mpam_props *parent,
1976			     struct mpam_props *child, bool alias)
1977{
1978	if (CAN_MERGE_FEAT(parent, child, mpam_feat_cpor_part, alias)) {
1979		parent->cpbm_wd = child->cpbm_wd;
1980	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_cpor_part,
1981				   cpbm_wd, alias)) {
1982		pr_debug("cleared cpor_part\n");
1983		mpam_clear_feature(mpam_feat_cpor_part, parent);
1984		parent->cpbm_wd = 0;
1985	}
1986
1987	if (CAN_MERGE_FEAT(parent, child, mpam_feat_mbw_part, alias)) {
1988		parent->mbw_pbm_bits = child->mbw_pbm_bits;
1989	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_mbw_part,
1990				   mbw_pbm_bits, alias)) {
1991		pr_debug("cleared mbw_part\n");
1992		mpam_clear_feature(mpam_feat_mbw_part, parent);
1993		parent->mbw_pbm_bits = 0;
1994	}
1995
1996	/* bwa_wd is a count of bits, fewer bits means less precision */
1997	if (alias && !mpam_has_bwa_wd_feature(parent) &&
1998	    mpam_has_bwa_wd_feature(child)) {
1999		parent->bwa_wd = child->bwa_wd;
2000	} else if (MISMATCHED_HELPER(parent, child, mpam_has_bwa_wd_feature,
2001				     bwa_wd, alias)) {
2002		pr_debug("took the min bwa_wd\n");
2003		parent->bwa_wd = min(parent->bwa_wd, child->bwa_wd);
2004	}
2005
2006	if (alias && !mpam_has_cmax_wd_feature(parent) && mpam_has_cmax_wd_feature(child)) {
2007		parent->cmax_wd = child->cmax_wd;
2008	} else if (MISMATCHED_HELPER(parent, child, mpam_has_cmax_wd_feature,
2009				     cmax_wd, alias)) {
2010		pr_debug("%s took the min cmax_wd\n", __func__);
2011		parent->cmax_wd = min(parent->cmax_wd, child->cmax_wd);
2012	}
2013
2014	if (CAN_MERGE_FEAT(parent, child, mpam_feat_cmax_cassoc, alias)) {
2015		parent->cassoc_wd = child->cassoc_wd;
2016	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_cmax_cassoc,
2017				   cassoc_wd, alias)) {
2018		pr_debug("%s cleared cassoc_wd\n", __func__);
2019		mpam_clear_feature(mpam_feat_cmax_cassoc, parent);
2020		parent->cassoc_wd = 0;
2021	}
2022
2023	/* For num properties, take the minimum */
2024	if (CAN_MERGE_FEAT(parent, child, mpam_feat_msmon_csu, alias)) {
2025		parent->num_csu_mon = child->num_csu_mon;
2026	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_msmon_csu,
2027				   num_csu_mon, alias)) {
2028		pr_debug("took the min num_csu_mon\n");
2029		parent->num_csu_mon = min(parent->num_csu_mon,
2030					  child->num_csu_mon);
2031	}
2032
2033	if (CAN_MERGE_FEAT(parent, child, mpam_feat_msmon_mbwu, alias)) {
2034		parent->num_mbwu_mon = child->num_mbwu_mon;
2035	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_msmon_mbwu,
2036				   num_mbwu_mon, alias)) {
2037		pr_debug("took the min num_mbwu_mon\n");
2038		parent->num_mbwu_mon = min(parent->num_mbwu_mon,
2039					   child->num_mbwu_mon);
2040	}
2041
2042	if (CAN_MERGE_FEAT(parent, child, mpam_feat_intpri_part, alias)) {
2043		parent->intpri_wd = child->intpri_wd;
2044	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_intpri_part,
2045				   intpri_wd, alias)) {
2046		pr_debug("%s took the min intpri_wd\n", __func__);
2047		parent->intpri_wd = min(parent->intpri_wd, child->intpri_wd);
2048	}
2049
2050	if (CAN_MERGE_FEAT(parent, child, mpam_feat_dspri_part, alias)) {
2051		parent->dspri_wd = child->dspri_wd;
2052	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_dspri_part,
2053				   dspri_wd, alias)) {
2054		pr_debug("%s took the min dspri_wd\n", __func__);
2055		parent->dspri_wd = min(parent->dspri_wd, child->dspri_wd);
2056	}
2057
2058	/* TODO: alias support for these two */
2059	/* {int,ds}pri may not have differing 0-low behaviour */
2060	if (mpam_has_feature(mpam_feat_intpri_part, parent) &&
2061	    (!mpam_has_feature(mpam_feat_intpri_part, child) ||
2062	     mpam_has_feature(mpam_feat_intpri_part_0_low, parent) !=
2063	     mpam_has_feature(mpam_feat_intpri_part_0_low, child))) {
2064		pr_debug("%s cleared intpri_part\n", __func__);
2065		mpam_clear_feature(mpam_feat_intpri_part, parent);
2066		mpam_clear_feature(mpam_feat_intpri_part_0_low, parent);
2067	}
2068	if (mpam_has_feature(mpam_feat_dspri_part, parent) &&
2069	    (!mpam_has_feature(mpam_feat_dspri_part, child) ||
2070	     mpam_has_feature(mpam_feat_dspri_part_0_low, parent) !=
2071	     mpam_has_feature(mpam_feat_dspri_part_0_low, child))) {
2072		pr_debug("%s cleared dspri_part\n", __func__);
2073		mpam_clear_feature(mpam_feat_dspri_part, parent);
2074		mpam_clear_feature(mpam_feat_dspri_part_0_low, parent);
2075	}
2076
2077	if (alias) {
2078		/* Merge features for aliased resources */
2079		bitmap_or(parent->features, parent->features, child->features, MPAM_FEATURE_LAST);
2080	} else {
2081		/* Clear missing features for non aliasing */
2082		bitmap_and(parent->features, parent->features, child->features, MPAM_FEATURE_LAST);
2083	}
2084}
2085
2086/*
2087 * If a vmsc doesn't match class feature/configuration, do the right thing(tm).
2088 * For 'num' properties we can just take the minimum.
2089 * For properties where the mismatched unused bits would make a difference, we
2090 * nobble the class feature, as we can't configure all the resources.
2091 * e.g. The L3 cache is composed of two resources with 13 and 17 portion
2092 * bitmaps respectively.
2093 */
2094static void
2095__class_props_mismatch(struct mpam_class *class, struct mpam_vmsc *vmsc)
2096{
2097	struct mpam_props *cprops = &class->props;
2098	struct mpam_props *vprops = &vmsc->props;
2099	struct device *dev = &vmsc->msc->pdev->dev;
2100
2101	lockdep_assert_held(&mpam_list_lock); /* we modify class */
2102
2103	dev_dbg(dev, "Merging features for class:0x%lx &= vmsc:0x%lx\n",
2104		(long)cprops->features, (long)vprops->features);
2105
2106	/* Take the safe value for any common features */
2107	__props_mismatch(cprops, vprops, false);
2108}
2109
2110static void
2111__vmsc_props_mismatch(struct mpam_vmsc *vmsc, struct mpam_msc_ris *ris)
2112{
2113	struct mpam_props *rprops = &ris->props;
2114	struct mpam_props *vprops = &vmsc->props;
2115	struct device *dev = &vmsc->msc->pdev->dev;
2116
2117	lockdep_assert_held(&mpam_list_lock); /* we modify vmsc */
2118
2119	dev_dbg(dev, "Merging features for vmsc:0x%lx |= ris:0x%lx\n",
2120		(long)vprops->features, (long)rprops->features);
2121
2122	/*
2123	 * Merge mismatched features - Copy any features that aren't common,
2124	 * but take the safe value for any common features.
2125	 */
2126	__props_mismatch(vprops, rprops, true);
2127}
2128
2129/*
2130 * Copy the first component's first vMSC's properties and features to the
2131 * class. __class_props_mismatch() will remove conflicts.
2132 * It is not possible to have a class with no components, or a component with
2133 * no resources. The vMSC properties have already been built.
2134 */
2135static void mpam_enable_init_class_features(struct mpam_class *class)
2136{
2137	struct mpam_vmsc *vmsc;
2138	struct mpam_component *comp;
2139
2140	comp = list_first_entry(&class->components,
2141				struct mpam_component, class_list);
2142	vmsc = list_first_entry(&comp->vmsc,
2143				struct mpam_vmsc, comp_list);
2144
2145	class->props = vmsc->props;
2146}
2147
2148static void mpam_enable_merge_vmsc_features(struct mpam_component *comp)
2149{
2150	struct mpam_vmsc *vmsc;
2151	struct mpam_msc_ris *ris;
2152	struct mpam_class *class = comp->class;
2153
2154	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
2155		list_for_each_entry(ris, &vmsc->ris, vmsc_list) {
2156			__vmsc_props_mismatch(vmsc, ris);
2157			class->nrdy_usec = max(class->nrdy_usec,
2158					       vmsc->msc->nrdy_usec);
2159		}
2160	}
2161}
2162
2163static void mpam_enable_merge_class_features(struct mpam_component *comp)
2164{
2165	struct mpam_vmsc *vmsc;
2166	struct mpam_class *class = comp->class;
2167
2168	list_for_each_entry(vmsc, &comp->vmsc, comp_list)
2169		__class_props_mismatch(class, vmsc);
2170}
2171
2172/*
2173 * Merge all the common resource features into class.
2174 * vmsc features are bitwise-or'd together by mpam_enable_merge_vmsc_features()
2175 * as the first step so that mpam_enable_init_class_features() can initialise
2176 * the class with a representative set of features.
2177 * Next the mpam_enable_merge_class_features() bitwise-and's all the vmsc
2178 * features to form the class features.
2179 * Other features are the min/max as appropriate.
2180 *
2181 * To avoid walking the whole tree twice, the class->nrdy_usec property is
2182 * updated when working with the vmsc as it is a max(), and doesn't need
2183 * initialising first.
2184 */
2185static void mpam_enable_merge_features(struct list_head *all_classes_list)
2186{
2187	struct mpam_class *class;
2188	struct mpam_component *comp;
2189
2190	lockdep_assert_held(&mpam_list_lock);
2191
2192	list_for_each_entry(class, all_classes_list, classes_list) {
2193		list_for_each_entry(comp, &class->components, class_list)
2194			mpam_enable_merge_vmsc_features(comp);
2195
2196		mpam_enable_init_class_features(class);
2197
2198		list_for_each_entry(comp, &class->components, class_list)
2199			mpam_enable_merge_class_features(comp);
2200	}
2201}
2202
2203static char *mpam_errcode_names[16] = {
2204	[MPAM_ERRCODE_NONE]			= "No error",
2205	[MPAM_ERRCODE_PARTID_SEL_RANGE]		= "PARTID_SEL_Range",
2206	[MPAM_ERRCODE_REQ_PARTID_RANGE]		= "Req_PARTID_Range",
2207	[MPAM_ERRCODE_MSMONCFG_ID_RANGE]	= "MSMONCFG_ID_RANGE",
2208	[MPAM_ERRCODE_REQ_PMG_RANGE]		= "Req_PMG_Range",
2209	[MPAM_ERRCODE_MONITOR_RANGE]		= "Monitor_Range",
2210	[MPAM_ERRCODE_INTPARTID_RANGE]		= "intPARTID_Range",
2211	[MPAM_ERRCODE_UNEXPECTED_INTERNAL]	= "Unexpected_INTERNAL",
2212	[MPAM_ERRCODE_UNDEFINED_RIS_PART_SEL]	= "Undefined_RIS_PART_SEL",
2213	[MPAM_ERRCODE_RIS_NO_CONTROL]		= "RIS_No_Control",
2214	[MPAM_ERRCODE_UNDEFINED_RIS_MON_SEL]	= "Undefined_RIS_MON_SEL",
2215	[MPAM_ERRCODE_RIS_NO_MONITOR]		= "RIS_No_Monitor",
2216	[12 ... 15] = "Reserved"
2217};
2218
2219static int mpam_enable_msc_ecr(void *_msc)
2220{
2221	struct mpam_msc *msc = _msc;
2222
2223	__mpam_write_reg(msc, MPAMF_ECR, MPAMF_ECR_INTEN);
2224
2225	return 0;
2226}
2227
2228/* This can run in mpam_disable(), and the interrupt handler on the same CPU */
2229static int mpam_disable_msc_ecr(void *_msc)
2230{
2231	struct mpam_msc *msc = _msc;
2232
2233	__mpam_write_reg(msc, MPAMF_ECR, 0);
2234
2235	return 0;
2236}
2237
2238static irqreturn_t __mpam_irq_handler(int irq, struct mpam_msc *msc)
2239{
2240	u64 reg;
2241	u16 partid;
2242	u8 errcode, pmg, ris;
2243
2244	if (WARN_ON_ONCE(!msc) ||
2245	    WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
2246					   &msc->accessibility)))
2247		return IRQ_NONE;
2248
2249	reg = mpam_msc_read_esr(msc);
2250
2251	errcode = FIELD_GET(MPAMF_ESR_ERRCODE, reg);
2252	if (!errcode)
2253		return IRQ_NONE;
2254
2255	/* Clear level triggered irq */
2256	mpam_msc_clear_esr(msc);
2257
2258	partid = FIELD_GET(MPAMF_ESR_PARTID_MON, reg);
2259	pmg = FIELD_GET(MPAMF_ESR_PMG, reg);
2260	ris = FIELD_GET(MPAMF_ESR_RIS, reg);
2261
2262	pr_err_ratelimited("error irq from msc:%u '%s', partid:%u, pmg: %u, ris: %u\n",
2263			   msc->id, mpam_errcode_names[errcode], partid, pmg,
2264			   ris);
2265
2266	/* Disable this interrupt. */
2267	mpam_disable_msc_ecr(msc);
2268
2269	/* Are we racing with the thread disabling MPAM? */
2270	if (!mpam_is_enabled())
2271		return IRQ_HANDLED;
2272
2273	/*
2274	 * Schedule the teardown work. Don't use a threaded IRQ as we can't
2275	 * unregister the interrupt from the threaded part of the handler.
2276	 */
2277	mpam_disable_reason = "hardware error interrupt";
2278	schedule_work(&mpam_broken_work);
2279
2280	return IRQ_HANDLED;
2281}
2282
2283static irqreturn_t mpam_ppi_handler(int irq, void *dev_id)
2284{
2285	struct mpam_msc *msc = *(struct mpam_msc **)dev_id;
2286
2287	return __mpam_irq_handler(irq, msc);
2288}
2289
2290static irqreturn_t mpam_spi_handler(int irq, void *dev_id)
2291{
2292	struct mpam_msc *msc = dev_id;
2293
2294	return __mpam_irq_handler(irq, msc);
2295}
2296
2297static int mpam_register_irqs(void)
2298{
2299	int err, irq;
2300	struct mpam_msc *msc;
2301
2302	lockdep_assert_cpus_held();
2303
2304	guard(srcu)(&mpam_srcu);
2305	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
2306				 srcu_read_lock_held(&mpam_srcu)) {
2307		irq = platform_get_irq_byname_optional(msc->pdev, "error");
2308		if (irq <= 0)
2309			continue;
2310
2311		/* The MPAM spec says the interrupt can be SPI, PPI or LPI */
2312		/* We anticipate sharing the interrupt with other MSCs */
2313		if (irq_is_percpu(irq)) {
2314			err = request_percpu_irq(irq, &mpam_ppi_handler,
2315						 "mpam:msc:error",
2316						 msc->error_dev_id);
2317			if (err)
2318				return err;
2319
2320			msc->reenable_error_ppi = irq;
2321			smp_call_function_many(&msc->accessibility,
2322					       &_enable_percpu_irq, &irq,
2323					       true);
2324		} else {
2325			err = devm_request_irq(&msc->pdev->dev, irq,
2326					       &mpam_spi_handler, IRQF_SHARED,
2327					       "mpam:msc:error", msc);
2328			if (err)
2329				return err;
2330		}
2331
2332		mutex_lock(&msc->error_irq_lock);
2333		msc->error_irq_req = true;
2334		mpam_touch_msc(msc, mpam_enable_msc_ecr, msc);
2335		msc->error_irq_hw_enabled = true;
2336		mutex_unlock(&msc->error_irq_lock);
2337	}
2338
2339	return 0;
2340}
2341
2342static void mpam_unregister_irqs(void)
2343{
2344	int irq;
2345	struct mpam_msc *msc;
2346
2347	guard(cpus_read_lock)();
2348	guard(srcu)(&mpam_srcu);
2349	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
2350				 srcu_read_lock_held(&mpam_srcu)) {
2351		irq = platform_get_irq_byname_optional(msc->pdev, "error");
2352		if (irq <= 0)
2353			continue;
2354
2355		mutex_lock(&msc->error_irq_lock);
2356		if (msc->error_irq_hw_enabled) {
2357			mpam_touch_msc(msc, mpam_disable_msc_ecr, msc);
2358			msc->error_irq_hw_enabled = false;
2359		}
2360
2361		if (msc->error_irq_req) {
2362			if (irq_is_percpu(irq)) {
2363				msc->reenable_error_ppi = 0;
2364				free_percpu_irq(irq, msc->error_dev_id);
2365			} else {
2366				devm_free_irq(&msc->pdev->dev, irq, msc);
2367			}
2368			msc->error_irq_req = false;
2369		}
2370		mutex_unlock(&msc->error_irq_lock);
2371	}
2372}
2373
2374static void __destroy_component_cfg(struct mpam_component *comp)
2375{
2376	struct mpam_msc *msc;
2377	struct mpam_vmsc *vmsc;
2378	struct mpam_msc_ris *ris;
2379
2380	lockdep_assert_held(&mpam_list_lock);
2381
2382	add_to_garbage(comp->cfg);
2383	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
2384		msc = vmsc->msc;
2385
2386		if (mpam_mon_sel_lock(msc)) {
2387			list_for_each_entry(ris, &vmsc->ris, vmsc_list)
2388				add_to_garbage(ris->mbwu_state);
2389			mpam_mon_sel_unlock(msc);
2390		}
2391	}
2392}
2393
2394static void mpam_reset_component_cfg(struct mpam_component *comp)
2395{
2396	int i;
2397	struct mpam_props *cprops = &comp->class->props;
2398
2399	mpam_assert_partid_sizes_fixed();
2400
2401	if (!comp->cfg)
2402		return;
2403
2404	for (i = 0; i <= mpam_partid_max; i++) {
2405		comp->cfg[i] = (struct mpam_config) {};
2406		if (cprops->cpbm_wd)
2407			comp->cfg[i].cpbm = GENMASK(cprops->cpbm_wd - 1, 0);
2408		if (cprops->mbw_pbm_bits)
2409			comp->cfg[i].mbw_pbm = GENMASK(cprops->mbw_pbm_bits - 1, 0);
2410		if (cprops->bwa_wd)
2411			comp->cfg[i].mbw_max = GENMASK(15, 16 - cprops->bwa_wd);
2412	}
2413}
2414
2415static int __allocate_component_cfg(struct mpam_component *comp)
2416{
2417	struct mpam_vmsc *vmsc;
2418
2419	mpam_assert_partid_sizes_fixed();
2420
2421	if (comp->cfg)
2422		return 0;
2423
2424	comp->cfg = kcalloc(mpam_partid_max + 1, sizeof(*comp->cfg), GFP_KERNEL);
2425	if (!comp->cfg)
2426		return -ENOMEM;
2427
2428	/*
2429	 * The array is free()d in one go, so only cfg[0]'s structure needs
2430	 * to be initialised.
2431	 */
2432	init_garbage(&comp->cfg[0].garbage);
2433
2434	mpam_reset_component_cfg(comp);
2435
2436	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
2437		struct mpam_msc *msc;
2438		struct mpam_msc_ris *ris;
2439		struct msmon_mbwu_state *mbwu_state;
2440
2441		if (!vmsc->props.num_mbwu_mon)
2442			continue;
2443
2444		msc = vmsc->msc;
2445		list_for_each_entry(ris, &vmsc->ris, vmsc_list) {
2446			if (!ris->props.num_mbwu_mon)
2447				continue;
2448
2449			mbwu_state = kcalloc(ris->props.num_mbwu_mon,
2450					     sizeof(*ris->mbwu_state),
2451					     GFP_KERNEL);
2452			if (!mbwu_state) {
2453				__destroy_component_cfg(comp);
2454				return -ENOMEM;
2455			}
2456
2457			init_garbage(&mbwu_state[0].garbage);
2458
2459			if (mpam_mon_sel_lock(msc)) {
2460				ris->mbwu_state = mbwu_state;
2461				mpam_mon_sel_unlock(msc);
2462			}
2463		}
2464	}
2465
2466	return 0;
2467}
2468
2469static int mpam_allocate_config(void)
2470{
2471	struct mpam_class *class;
2472	struct mpam_component *comp;
2473
2474	lockdep_assert_held(&mpam_list_lock);
2475
2476	list_for_each_entry(class, &mpam_classes, classes_list) {
2477		list_for_each_entry(comp, &class->components, class_list) {
2478			int err = __allocate_component_cfg(comp);
2479			if (err)
2480				return err;
2481		}
2482	}
2483
2484	return 0;
2485}
2486
2487static void mpam_enable_once(void)
2488{
2489	int err;
2490
2491	/*
2492	 * Once the cpuhp callbacks have been changed, mpam_partid_max can no
2493	 * longer change.
2494	 */
2495	spin_lock(&partid_max_lock);
2496	partid_max_published = true;
2497	spin_unlock(&partid_max_lock);
2498
2499	/*
2500	 * If all the MSC have been probed, enabling the IRQs happens next.
2501	 * That involves cross-calling to a CPU that can reach the MSC, and
2502	 * the locks must be taken in this order:
2503	 */
2504	cpus_read_lock();
2505	mutex_lock(&mpam_list_lock);
2506	do {
2507		mpam_enable_merge_features(&mpam_classes);
2508
2509		err = mpam_register_irqs();
2510		if (err) {
2511			pr_warn("Failed to register irqs: %d\n", err);
2512			break;
2513		}
2514
2515		err = mpam_allocate_config();
2516		if (err) {
2517			pr_err("Failed to allocate configuration arrays.\n");
2518			break;
2519		}
2520	} while (0);
2521	mutex_unlock(&mpam_list_lock);
2522	cpus_read_unlock();
2523
2524	if (err) {
2525		mpam_disable_reason = "Failed to enable.";
2526		schedule_work(&mpam_broken_work);
2527		return;
2528	}
2529
2530	static_branch_enable(&mpam_enabled);
2531	mpam_register_cpuhp_callbacks(mpam_cpu_online, mpam_cpu_offline,
2532				      "mpam:online");
2533
2534	/* Use printk() to avoid the pr_fmt adding the function name. */
2535	printk(KERN_INFO "MPAM enabled with %u PARTIDs and %u PMGs\n",
2536	       mpam_partid_max + 1, mpam_pmg_max + 1);
2537}
2538
2539static void mpam_reset_component_locked(struct mpam_component *comp)
2540{
2541	struct mpam_vmsc *vmsc;
2542
2543	lockdep_assert_cpus_held();
2544	mpam_assert_partid_sizes_fixed();
2545
2546	mpam_reset_component_cfg(comp);
2547
2548	guard(srcu)(&mpam_srcu);
2549	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
2550				 srcu_read_lock_held(&mpam_srcu)) {
2551		struct mpam_msc *msc = vmsc->msc;
2552		struct mpam_msc_ris *ris;
2553
2554		list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
2555					 srcu_read_lock_held(&mpam_srcu)) {
2556			if (!ris->in_reset_state)
2557				mpam_touch_msc(msc, mpam_reset_ris, ris);
2558			ris->in_reset_state = true;
2559		}
2560	}
2561}
2562
2563static void mpam_reset_class_locked(struct mpam_class *class)
2564{
2565	struct mpam_component *comp;
2566
2567	lockdep_assert_cpus_held();
2568
2569	guard(srcu)(&mpam_srcu);
2570	list_for_each_entry_srcu(comp, &class->components, class_list,
2571				 srcu_read_lock_held(&mpam_srcu))
2572		mpam_reset_component_locked(comp);
2573}
2574
2575static void mpam_reset_class(struct mpam_class *class)
2576{
2577	cpus_read_lock();
2578	mpam_reset_class_locked(class);
2579	cpus_read_unlock();
2580}
2581
2582/*
2583 * Called in response to an error IRQ.
2584 * All of MPAMs errors indicate a software bug, restore any modified
2585 * controls to their reset values.
2586 */
2587void mpam_disable(struct work_struct *ignored)
2588{
2589	int idx;
2590	struct mpam_class *class;
2591	struct mpam_msc *msc, *tmp;
2592
2593	mutex_lock(&mpam_cpuhp_state_lock);
2594	if (mpam_cpuhp_state) {
2595		cpuhp_remove_state(mpam_cpuhp_state);
2596		mpam_cpuhp_state = 0;
2597	}
2598	mutex_unlock(&mpam_cpuhp_state_lock);
2599
2600	static_branch_disable(&mpam_enabled);
2601
2602	mpam_unregister_irqs();
2603
2604	idx = srcu_read_lock(&mpam_srcu);
2605	list_for_each_entry_srcu(class, &mpam_classes, classes_list,
2606				 srcu_read_lock_held(&mpam_srcu))
2607		mpam_reset_class(class);
2608	srcu_read_unlock(&mpam_srcu, idx);
2609
2610	mutex_lock(&mpam_list_lock);
2611	list_for_each_entry_safe(msc, tmp, &mpam_all_msc, all_msc_list)
2612		mpam_msc_destroy(msc);
2613	mutex_unlock(&mpam_list_lock);
2614	mpam_free_garbage();
2615
2616	pr_err_once("MPAM disabled due to %s\n", mpam_disable_reason);
2617}
2618
2619/*
2620 * Enable mpam once all devices have been probed.
2621 * Scheduled by mpam_discovery_cpu_online() once all devices have been created.
2622 * Also scheduled when new devices are probed when new CPUs come online.
2623 */
2624void mpam_enable(struct work_struct *work)
2625{
2626	static atomic_t once;
2627	struct mpam_msc *msc;
2628	bool all_devices_probed = true;
2629
2630	/* Have we probed all the hw devices? */
2631	guard(srcu)(&mpam_srcu);
2632	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
2633				 srcu_read_lock_held(&mpam_srcu)) {
2634		mutex_lock(&msc->probe_lock);
2635		if (!msc->probed)
2636			all_devices_probed = false;
2637		mutex_unlock(&msc->probe_lock);
2638
2639		if (!all_devices_probed)
2640			break;
2641	}
2642
2643	if (all_devices_probed && !atomic_fetch_inc(&once))
2644		mpam_enable_once();
2645}
2646
2647#define maybe_update_config(cfg, feature, newcfg, member, changes) do { \
2648	if (mpam_has_feature(feature, newcfg) &&			\
2649	    (newcfg)->member != (cfg)->member) {			\
2650		(cfg)->member = (newcfg)->member;			\
2651		mpam_set_feature(feature, cfg);				\
2652									\
2653		(changes) = true;					\
2654	}								\
2655} while (0)
2656
2657static bool mpam_update_config(struct mpam_config *cfg,
2658			       const struct mpam_config *newcfg)
2659{
2660	bool has_changes = false;
2661
2662	maybe_update_config(cfg, mpam_feat_cpor_part, newcfg, cpbm, has_changes);
2663	maybe_update_config(cfg, mpam_feat_mbw_part, newcfg, mbw_pbm, has_changes);
2664	maybe_update_config(cfg, mpam_feat_mbw_max, newcfg, mbw_max, has_changes);
2665
2666	return has_changes;
2667}
2668
2669int mpam_apply_config(struct mpam_component *comp, u16 partid,
2670		      struct mpam_config *cfg)
2671{
2672	struct mpam_write_config_arg arg;
2673	struct mpam_msc_ris *ris;
2674	struct mpam_vmsc *vmsc;
2675	struct mpam_msc *msc;
2676
2677	lockdep_assert_cpus_held();
2678
2679	/* Don't pass in the current config! */
2680	WARN_ON_ONCE(&comp->cfg[partid] == cfg);
2681
2682	if (!mpam_update_config(&comp->cfg[partid], cfg))
2683		return 0;
2684
2685	arg.comp = comp;
2686	arg.partid = partid;
2687
2688	guard(srcu)(&mpam_srcu);
2689	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
2690				 srcu_read_lock_held(&mpam_srcu)) {
2691		msc = vmsc->msc;
2692
2693		mutex_lock(&msc->cfg_lock);
2694		list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
2695					 srcu_read_lock_held(&mpam_srcu)) {
2696			arg.ris = ris;
2697			mpam_touch_msc(msc, __write_config, &arg);
2698		}
2699		mutex_unlock(&msc->cfg_lock);
2700	}
2701
2702	return 0;
2703}
2704
2705static int __init mpam_msc_driver_init(void)
2706{
2707	if (!system_supports_mpam())
2708		return -EOPNOTSUPP;
2709
2710	init_srcu_struct(&mpam_srcu);
2711
2712	fw_num_msc = acpi_mpam_count_msc();
2713	if (fw_num_msc <= 0) {
2714		pr_err("No MSC devices found in firmware\n");
2715		return -EINVAL;
2716	}
2717
2718	return platform_driver_register(&mpam_msc_driver);
2719}
2720
2721/* Must occur after arm64_mpam_register_cpus() from arch_initcall() */
2722subsys_initcall(mpam_msc_driver_init);
2723
2724#ifdef CONFIG_MPAM_KUNIT_TEST
2725#include "test_mpam_devices.c"
2726#endif