core.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Resource Director Technology(RDT)
   4 * - Cache Allocation code.
   5 *
   6 * Copyright (C) 2016 Intel Corporation
   7 *
   8 * Authors:
   9 *    Fenghua Yu <fenghua.yu@intel.com>
  10 *    Tony Luck <tony.luck@intel.com>
  11 *    Vikas Shivappa <vikas.shivappa@intel.com>
  12 *
  13 * More information about RDT be found in the Intel (R) x86 Architecture
  14 * Software Developer Manual June 2016, volume 3, section 17.17.
  15 */
  16
  17#define pr_fmt(fmt)	"resctrl: " fmt
  18
  19#include <linux/cpu.h>
  20#include <linux/slab.h>
  21#include <linux/err.h>
  22#include <linux/cpuhotplug.h>
  23
  24#include <asm/cpu_device_id.h>
  25#include <asm/msr.h>
  26#include <asm/resctrl.h>
  27#include "internal.h"
  28
  29/*
  30 * rdt_domain structures are kfree()d when their last CPU goes offline,
  31 * and allocated when the first CPU in a new domain comes online.
  32 * The rdt_resource's domain list is updated when this happens. Readers of
  33 * the domain list must either take cpus_read_lock(), or rely on an RCU
  34 * read-side critical section, to avoid observing concurrent modification.
  35 * All writers take this mutex:
  36 */
  37static DEFINE_MUTEX(domain_list_lock);
  38
  39/*
  40 * The cached resctrl_pqr_state is strictly per CPU and can never be
  41 * updated from a remote CPU. Functions which modify the state
  42 * are called with interrupts disabled and no preemption, which
  43 * is sufficient for the protection.
  44 */
  45DEFINE_PER_CPU(struct resctrl_pqr_state, pqr_state);
  46
  47/*
  48 * Global boolean for rdt_alloc which is true if any
  49 * resource allocation is enabled.
  50 */
  51bool rdt_alloc_capable;
  52
  53static void mba_wrmsr_intel(struct msr_param *m);
  54static void cat_wrmsr(struct msr_param *m);
  55static void mba_wrmsr_amd(struct msr_param *m);
  56
  57#define ctrl_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.ctrl_domains)
  58#define mon_domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.mon_domains)
  59
  60struct rdt_hw_resource rdt_resources_all[RDT_NUM_RESOURCES] = {
  61	[RDT_RESOURCE_L3] =
  62	{
  63		.r_resctrl = {
  64			.name			= "L3",
  65			.ctrl_scope		= RESCTRL_L3_CACHE,
  66			.mon_scope		= RESCTRL_L3_CACHE,
  67			.ctrl_domains		= ctrl_domain_init(RDT_RESOURCE_L3),
  68			.mon_domains		= mon_domain_init(RDT_RESOURCE_L3),
  69			.schema_fmt		= RESCTRL_SCHEMA_BITMAP,
  70		},
  71		.msr_base		= MSR_IA32_L3_CBM_BASE,
  72		.msr_update		= cat_wrmsr,
  73	},
  74	[RDT_RESOURCE_L2] =
  75	{
  76		.r_resctrl = {
  77			.name			= "L2",
  78			.ctrl_scope		= RESCTRL_L2_CACHE,
  79			.ctrl_domains		= ctrl_domain_init(RDT_RESOURCE_L2),
  80			.schema_fmt		= RESCTRL_SCHEMA_BITMAP,
  81		},
  82		.msr_base		= MSR_IA32_L2_CBM_BASE,
  83		.msr_update		= cat_wrmsr,
  84	},
  85	[RDT_RESOURCE_MBA] =
  86	{
  87		.r_resctrl = {
  88			.name			= "MB",
  89			.ctrl_scope		= RESCTRL_L3_CACHE,
  90			.ctrl_domains		= ctrl_domain_init(RDT_RESOURCE_MBA),
  91			.schema_fmt		= RESCTRL_SCHEMA_RANGE,
  92		},
  93	},
  94	[RDT_RESOURCE_SMBA] =
  95	{
  96		.r_resctrl = {
  97			.name			= "SMBA",
  98			.ctrl_scope		= RESCTRL_L3_CACHE,
  99			.ctrl_domains		= ctrl_domain_init(RDT_RESOURCE_SMBA),
 100			.schema_fmt		= RESCTRL_SCHEMA_RANGE,
 101		},
 102	},
 103};
 104
 105u32 resctrl_arch_system_num_rmid_idx(void)
 106{
 107	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
 108
 109	/* RMID are independent numbers for x86. num_rmid_idx == num_rmid */
 110	return r->mon.num_rmid;
 111}
 112
 113struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l)
 114{
 115	if (l >= RDT_NUM_RESOURCES)
 116		return NULL;
 117
 118	return &rdt_resources_all[l].r_resctrl;
 119}
 120
 121/*
 122 * cache_alloc_hsw_probe() - Have to probe for Intel haswell server CPUs
 123 * as they do not have CPUID enumeration support for Cache allocation.
 124 * The check for Vendor/Family/Model is not enough to guarantee that
 125 * the MSRs won't #GP fault because only the following SKUs support
 126 * CAT:
 127 *	Intel(R) Xeon(R)  CPU E5-2658  v3  @  2.20GHz
 128 *	Intel(R) Xeon(R)  CPU E5-2648L v3  @  1.80GHz
 129 *	Intel(R) Xeon(R)  CPU E5-2628L v3  @  2.00GHz
 130 *	Intel(R) Xeon(R)  CPU E5-2618L v3  @  2.30GHz
 131 *	Intel(R) Xeon(R)  CPU E5-2608L v3  @  2.00GHz
 132 *	Intel(R) Xeon(R)  CPU E5-2658A v3  @  2.20GHz
 133 *
 134 * Probe by trying to write the first of the L3 cache mask registers
 135 * and checking that the bits stick. Max CLOSids is always 4 and max cbm length
 136 * is always 20 on hsw server parts. The minimum cache bitmask length
 137 * allowed for HSW server is always 2 bits. Hardcode all of them.
 138 */
 139static inline void cache_alloc_hsw_probe(void)
 140{
 141	struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_L3];
 142	struct rdt_resource *r  = &hw_res->r_resctrl;
 143	u64 max_cbm = BIT_ULL_MASK(20) - 1, l3_cbm_0;
 144
 145	if (wrmsrq_safe(MSR_IA32_L3_CBM_BASE, max_cbm))
 146		return;
 147
 148	rdmsrq(MSR_IA32_L3_CBM_BASE, l3_cbm_0);
 149
 150	/* If all the bits were set in MSR, return success */
 151	if (l3_cbm_0 != max_cbm)
 152		return;
 153
 154	hw_res->num_closid = 4;
 155	r->cache.cbm_len = 20;
 156	r->cache.shareable_bits = 0xc0000;
 157	r->cache.min_cbm_bits = 2;
 158	r->cache.arch_has_sparse_bitmasks = false;
 159	r->alloc_capable = true;
 160
 161	rdt_alloc_capable = true;
 162}
 163
 164/*
 165 * rdt_get_mb_table() - get a mapping of bandwidth(b/w) percentage values
 166 * exposed to user interface and the h/w understandable delay values.
 167 *
 168 * The non-linear delay values have the granularity of power of two
 169 * and also the h/w does not guarantee a curve for configured delay
 170 * values vs. actual b/w enforced.
 171 * Hence we need a mapping that is pre calibrated so the user can
 172 * express the memory b/w as a percentage value.
 173 */
 174static inline bool rdt_get_mb_table(struct rdt_resource *r)
 175{
 176	/*
 177	 * There are no Intel SKUs as of now to support non-linear delay.
 178	 */
 179	pr_info("MBA b/w map not implemented for cpu:%d, model:%d",
 180		boot_cpu_data.x86, boot_cpu_data.x86_model);
 181
 182	return false;
 183}
 184
 185static __init bool __get_mem_config_intel(struct rdt_resource *r)
 186{
 187	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 188	union cpuid_0x10_3_eax eax;
 189	union cpuid_0x10_x_edx edx;
 190	u32 ebx, ecx, max_delay;
 191
 192	cpuid_count(0x00000010, 3, &eax.full, &ebx, &ecx, &edx.full);
 193	hw_res->num_closid = edx.split.cos_max + 1;
 194	max_delay = eax.split.max_delay + 1;
 195	r->membw.max_bw = MAX_MBA_BW;
 196	r->membw.arch_needs_linear = true;
 197	if (ecx & MBA_IS_LINEAR) {
 198		r->membw.delay_linear = true;
 199		r->membw.min_bw = MAX_MBA_BW - max_delay;
 200		r->membw.bw_gran = MAX_MBA_BW - max_delay;
 201	} else {
 202		if (!rdt_get_mb_table(r))
 203			return false;
 204		r->membw.arch_needs_linear = false;
 205	}
 206
 207	if (boot_cpu_has(X86_FEATURE_PER_THREAD_MBA))
 208		r->membw.throttle_mode = THREAD_THROTTLE_PER_THREAD;
 209	else
 210		r->membw.throttle_mode = THREAD_THROTTLE_MAX;
 211
 212	r->alloc_capable = true;
 213
 214	return true;
 215}
 216
 217static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r)
 218{
 219	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 220	u32 eax, ebx, ecx, edx, subleaf;
 221
 222	/*
 223	 * Query CPUID_Fn80000020_EDX_x01 for MBA and
 224	 * CPUID_Fn80000020_EDX_x02 for SMBA
 225	 */
 226	subleaf = (r->rid == RDT_RESOURCE_SMBA) ? 2 :  1;
 227
 228	cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx);
 229	hw_res->num_closid = edx + 1;
 230	r->membw.max_bw = 1 << eax;
 231
 232	/* AMD does not use delay */
 233	r->membw.delay_linear = false;
 234	r->membw.arch_needs_linear = false;
 235
 236	/*
 237	 * AMD does not use memory delay throttle model to control
 238	 * the allocation like Intel does.
 239	 */
 240	r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED;
 241	r->membw.min_bw = 0;
 242	r->membw.bw_gran = 1;
 243
 244	r->alloc_capable = true;
 245
 246	return true;
 247}
 248
 249static void rdt_get_cache_alloc_cfg(int idx, struct rdt_resource *r)
 250{
 251	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 252	union cpuid_0x10_1_eax eax;
 253	union cpuid_0x10_x_ecx ecx;
 254	union cpuid_0x10_x_edx edx;
 255	u32 ebx, default_ctrl;
 256
 257	cpuid_count(0x00000010, idx, &eax.full, &ebx, &ecx.full, &edx.full);
 258	hw_res->num_closid = edx.split.cos_max + 1;
 259	r->cache.cbm_len = eax.split.cbm_len + 1;
 260	default_ctrl = BIT_MASK(eax.split.cbm_len + 1) - 1;
 261	r->cache.shareable_bits = ebx & default_ctrl;
 262	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
 263		r->cache.arch_has_sparse_bitmasks = ecx.split.noncont;
 264	r->alloc_capable = true;
 265}
 266
 267static void rdt_get_cdp_config(int level)
 268{
 269	/*
 270	 * By default, CDP is disabled. CDP can be enabled by mount parameter
 271	 * "cdp" during resctrl file system mount time.
 272	 */
 273	rdt_resources_all[level].cdp_enabled = false;
 274	rdt_resources_all[level].r_resctrl.cdp_capable = true;
 275}
 276
 277static void rdt_set_io_alloc_capable(struct rdt_resource *r)
 278{
 279	r->cache.io_alloc_capable = true;
 280}
 281
 282static void rdt_get_cdp_l3_config(void)
 283{
 284	rdt_get_cdp_config(RDT_RESOURCE_L3);
 285}
 286
 287static void rdt_get_cdp_l2_config(void)
 288{
 289	rdt_get_cdp_config(RDT_RESOURCE_L2);
 290}
 291
 292static void mba_wrmsr_amd(struct msr_param *m)
 293{
 294	struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom);
 295	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res);
 296	unsigned int i;
 297
 298	for (i = m->low; i < m->high; i++)
 299		wrmsrq(hw_res->msr_base + i, hw_dom->ctrl_val[i]);
 300}
 301
 302/*
 303 * Map the memory b/w percentage value to delay values
 304 * that can be written to QOS_MSRs.
 305 * There are currently no SKUs which support non linear delay values.
 306 */
 307static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r)
 308{
 309	if (r->membw.delay_linear)
 310		return MAX_MBA_BW - bw;
 311
 312	pr_warn_once("Non Linear delay-bw map not supported but queried\n");
 313	return MAX_MBA_BW;
 314}
 315
 316static void mba_wrmsr_intel(struct msr_param *m)
 317{
 318	struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom);
 319	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res);
 320	unsigned int i;
 321
 322	/*  Write the delay values for mba. */
 323	for (i = m->low; i < m->high; i++)
 324		wrmsrq(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], m->res));
 325}
 326
 327static void cat_wrmsr(struct msr_param *m)
 328{
 329	struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(m->dom);
 330	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res);
 331	unsigned int i;
 332
 333	for (i = m->low; i < m->high; i++)
 334		wrmsrq(hw_res->msr_base + i, hw_dom->ctrl_val[i]);
 335}
 336
 337u32 resctrl_arch_get_num_closid(struct rdt_resource *r)
 338{
 339	return resctrl_to_arch_res(r)->num_closid;
 340}
 341
 342void rdt_ctrl_update(void *arg)
 343{
 344	struct rdt_hw_resource *hw_res;
 345	struct msr_param *m = arg;
 346
 347	hw_res = resctrl_to_arch_res(m->res);
 348	hw_res->msr_update(m);
 349}
 350
 351static void setup_default_ctrlval(struct rdt_resource *r, u32 *dc)
 352{
 353	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 354	int i;
 355
 356	/*
 357	 * Initialize the Control MSRs to having no control.
 358	 * For Cache Allocation: Set all bits in cbm
 359	 * For Memory Allocation: Set b/w requested to 100%
 360	 */
 361	for (i = 0; i < hw_res->num_closid; i++, dc++)
 362		*dc = resctrl_get_default_ctrl(r);
 363}
 364
 365static void ctrl_domain_free(struct rdt_hw_ctrl_domain *hw_dom)
 366{
 367	kfree(hw_dom->ctrl_val);
 368	kfree(hw_dom);
 369}
 370
 371static void mon_domain_free(struct rdt_hw_mon_domain *hw_dom)
 372{
 373	int idx;
 374
 375	for_each_mbm_idx(idx)
 376		kfree(hw_dom->arch_mbm_states[idx]);
 377	kfree(hw_dom);
 378}
 379
 380static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_ctrl_domain *d)
 381{
 382	struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(d);
 383	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 384	struct msr_param m;
 385	u32 *dc;
 386
 387	dc = kmalloc_array(hw_res->num_closid, sizeof(*hw_dom->ctrl_val),
 388			   GFP_KERNEL);
 389	if (!dc)
 390		return -ENOMEM;
 391
 392	hw_dom->ctrl_val = dc;
 393	setup_default_ctrlval(r, dc);
 394
 395	m.res = r;
 396	m.dom = d;
 397	m.low = 0;
 398	m.high = hw_res->num_closid;
 399	hw_res->msr_update(&m);
 400	return 0;
 401}
 402
 403/**
 404 * arch_domain_mbm_alloc() - Allocate arch private storage for the MBM counters
 405 * @num_rmid:	The size of the MBM counter array
 406 * @hw_dom:	The domain that owns the allocated arrays
 407 */
 408static int arch_domain_mbm_alloc(u32 num_rmid, struct rdt_hw_mon_domain *hw_dom)
 409{
 410	size_t tsize = sizeof(*hw_dom->arch_mbm_states[0]);
 411	enum resctrl_event_id eventid;
 412	int idx;
 413
 414	for_each_mbm_event_id(eventid) {
 415		if (!resctrl_is_mon_event_enabled(eventid))
 416			continue;
 417		idx = MBM_STATE_IDX(eventid);
 418		hw_dom->arch_mbm_states[idx] = kcalloc(num_rmid, tsize, GFP_KERNEL);
 419		if (!hw_dom->arch_mbm_states[idx])
 420			goto cleanup;
 421	}
 422
 423	return 0;
 424cleanup:
 425	for_each_mbm_idx(idx) {
 426		kfree(hw_dom->arch_mbm_states[idx]);
 427		hw_dom->arch_mbm_states[idx] = NULL;
 428	}
 429
 430	return -ENOMEM;
 431}
 432
 433static int get_domain_id_from_scope(int cpu, enum resctrl_scope scope)
 434{
 435	switch (scope) {
 436	case RESCTRL_L2_CACHE:
 437	case RESCTRL_L3_CACHE:
 438		return get_cpu_cacheinfo_id(cpu, scope);
 439	case RESCTRL_L3_NODE:
 440		return cpu_to_node(cpu);
 441	default:
 442		break;
 443	}
 444
 445	return -EINVAL;
 446}
 447
 448static void domain_add_cpu_ctrl(int cpu, struct rdt_resource *r)
 449{
 450	int id = get_domain_id_from_scope(cpu, r->ctrl_scope);
 451	struct rdt_hw_ctrl_domain *hw_dom;
 452	struct list_head *add_pos = NULL;
 453	struct rdt_domain_hdr *hdr;
 454	struct rdt_ctrl_domain *d;
 455	int err;
 456
 457	lockdep_assert_held(&domain_list_lock);
 458
 459	if (id < 0) {
 460		pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n",
 461			     cpu, r->ctrl_scope, r->name);
 462		return;
 463	}
 464
 465	hdr = resctrl_find_domain(&r->ctrl_domains, id, &add_pos);
 466	if (hdr) {
 467		if (WARN_ON_ONCE(hdr->type != RESCTRL_CTRL_DOMAIN))
 468			return;
 469		d = container_of(hdr, struct rdt_ctrl_domain, hdr);
 470
 471		cpumask_set_cpu(cpu, &d->hdr.cpu_mask);
 472		if (r->cache.arch_has_per_cpu_cfg)
 473			rdt_domain_reconfigure_cdp(r);
 474		return;
 475	}
 476
 477	hw_dom = kzalloc_node(sizeof(*hw_dom), GFP_KERNEL, cpu_to_node(cpu));
 478	if (!hw_dom)
 479		return;
 480
 481	d = &hw_dom->d_resctrl;
 482	d->hdr.id = id;
 483	d->hdr.type = RESCTRL_CTRL_DOMAIN;
 484	cpumask_set_cpu(cpu, &d->hdr.cpu_mask);
 485
 486	rdt_domain_reconfigure_cdp(r);
 487
 488	if (domain_setup_ctrlval(r, d)) {
 489		ctrl_domain_free(hw_dom);
 490		return;
 491	}
 492
 493	list_add_tail_rcu(&d->hdr.list, add_pos);
 494
 495	err = resctrl_online_ctrl_domain(r, d);
 496	if (err) {
 497		list_del_rcu(&d->hdr.list);
 498		synchronize_rcu();
 499		ctrl_domain_free(hw_dom);
 500	}
 501}
 502
 503static void domain_add_cpu_mon(int cpu, struct rdt_resource *r)
 504{
 505	int id = get_domain_id_from_scope(cpu, r->mon_scope);
 506	struct list_head *add_pos = NULL;
 507	struct rdt_hw_mon_domain *hw_dom;
 508	struct rdt_domain_hdr *hdr;
 509	struct rdt_mon_domain *d;
 510	struct cacheinfo *ci;
 511	int err;
 512
 513	lockdep_assert_held(&domain_list_lock);
 514
 515	if (id < 0) {
 516		pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n",
 517			     cpu, r->mon_scope, r->name);
 518		return;
 519	}
 520
 521	hdr = resctrl_find_domain(&r->mon_domains, id, &add_pos);
 522	if (hdr) {
 523		if (WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN))
 524			return;
 525		d = container_of(hdr, struct rdt_mon_domain, hdr);
 526
 527		cpumask_set_cpu(cpu, &d->hdr.cpu_mask);
 528		/* Update the mbm_assign_mode state for the CPU if supported */
 529		if (r->mon.mbm_cntr_assignable)
 530			resctrl_arch_mbm_cntr_assign_set_one(r);
 531		return;
 532	}
 533
 534	hw_dom = kzalloc_node(sizeof(*hw_dom), GFP_KERNEL, cpu_to_node(cpu));
 535	if (!hw_dom)
 536		return;
 537
 538	d = &hw_dom->d_resctrl;
 539	d->hdr.id = id;
 540	d->hdr.type = RESCTRL_MON_DOMAIN;
 541	ci = get_cpu_cacheinfo_level(cpu, RESCTRL_L3_CACHE);
 542	if (!ci) {
 543		pr_warn_once("Can't find L3 cache for CPU:%d resource %s\n", cpu, r->name);
 544		mon_domain_free(hw_dom);
 545		return;
 546	}
 547	d->ci_id = ci->id;
 548	cpumask_set_cpu(cpu, &d->hdr.cpu_mask);
 549
 550	/* Update the mbm_assign_mode state for the CPU if supported */
 551	if (r->mon.mbm_cntr_assignable)
 552		resctrl_arch_mbm_cntr_assign_set_one(r);
 553
 554	arch_mon_domain_online(r, d);
 555
 556	if (arch_domain_mbm_alloc(r->mon.num_rmid, hw_dom)) {
 557		mon_domain_free(hw_dom);
 558		return;
 559	}
 560
 561	list_add_tail_rcu(&d->hdr.list, add_pos);
 562
 563	err = resctrl_online_mon_domain(r, d);
 564	if (err) {
 565		list_del_rcu(&d->hdr.list);
 566		synchronize_rcu();
 567		mon_domain_free(hw_dom);
 568	}
 569}
 570
 571static void domain_add_cpu(int cpu, struct rdt_resource *r)
 572{
 573	if (r->alloc_capable)
 574		domain_add_cpu_ctrl(cpu, r);
 575	if (r->mon_capable)
 576		domain_add_cpu_mon(cpu, r);
 577}
 578
 579static void domain_remove_cpu_ctrl(int cpu, struct rdt_resource *r)
 580{
 581	int id = get_domain_id_from_scope(cpu, r->ctrl_scope);
 582	struct rdt_hw_ctrl_domain *hw_dom;
 583	struct rdt_domain_hdr *hdr;
 584	struct rdt_ctrl_domain *d;
 585
 586	lockdep_assert_held(&domain_list_lock);
 587
 588	if (id < 0) {
 589		pr_warn_once("Can't find control domain id for CPU:%d scope:%d for resource %s\n",
 590			     cpu, r->ctrl_scope, r->name);
 591		return;
 592	}
 593
 594	hdr = resctrl_find_domain(&r->ctrl_domains, id, NULL);
 595	if (!hdr) {
 596		pr_warn("Can't find control domain for id=%d for CPU %d for resource %s\n",
 597			id, cpu, r->name);
 598		return;
 599	}
 600
 601	if (WARN_ON_ONCE(hdr->type != RESCTRL_CTRL_DOMAIN))
 602		return;
 603
 604	d = container_of(hdr, struct rdt_ctrl_domain, hdr);
 605	hw_dom = resctrl_to_arch_ctrl_dom(d);
 606
 607	cpumask_clear_cpu(cpu, &d->hdr.cpu_mask);
 608	if (cpumask_empty(&d->hdr.cpu_mask)) {
 609		resctrl_offline_ctrl_domain(r, d);
 610		list_del_rcu(&d->hdr.list);
 611		synchronize_rcu();
 612
 613		/*
 614		 * rdt_ctrl_domain "d" is going to be freed below, so clear
 615		 * its pointer from pseudo_lock_region struct.
 616		 */
 617		if (d->plr)
 618			d->plr->d = NULL;
 619		ctrl_domain_free(hw_dom);
 620
 621		return;
 622	}
 623}
 624
 625static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r)
 626{
 627	int id = get_domain_id_from_scope(cpu, r->mon_scope);
 628	struct rdt_hw_mon_domain *hw_dom;
 629	struct rdt_domain_hdr *hdr;
 630	struct rdt_mon_domain *d;
 631
 632	lockdep_assert_held(&domain_list_lock);
 633
 634	if (id < 0) {
 635		pr_warn_once("Can't find monitor domain id for CPU:%d scope:%d for resource %s\n",
 636			     cpu, r->mon_scope, r->name);
 637		return;
 638	}
 639
 640	hdr = resctrl_find_domain(&r->mon_domains, id, NULL);
 641	if (!hdr) {
 642		pr_warn("Can't find monitor domain for id=%d for CPU %d for resource %s\n",
 643			id, cpu, r->name);
 644		return;
 645	}
 646
 647	if (WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN))
 648		return;
 649
 650	d = container_of(hdr, struct rdt_mon_domain, hdr);
 651	hw_dom = resctrl_to_arch_mon_dom(d);
 652
 653	cpumask_clear_cpu(cpu, &d->hdr.cpu_mask);
 654	if (cpumask_empty(&d->hdr.cpu_mask)) {
 655		resctrl_offline_mon_domain(r, d);
 656		list_del_rcu(&d->hdr.list);
 657		synchronize_rcu();
 658		mon_domain_free(hw_dom);
 659
 660		return;
 661	}
 662}
 663
 664static void domain_remove_cpu(int cpu, struct rdt_resource *r)
 665{
 666	if (r->alloc_capable)
 667		domain_remove_cpu_ctrl(cpu, r);
 668	if (r->mon_capable)
 669		domain_remove_cpu_mon(cpu, r);
 670}
 671
 672static void clear_closid_rmid(int cpu)
 673{
 674	struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state);
 675
 676	state->default_closid = RESCTRL_RESERVED_CLOSID;
 677	state->default_rmid = RESCTRL_RESERVED_RMID;
 678	state->cur_closid = RESCTRL_RESERVED_CLOSID;
 679	state->cur_rmid = RESCTRL_RESERVED_RMID;
 680	wrmsr(MSR_IA32_PQR_ASSOC, RESCTRL_RESERVED_RMID,
 681	      RESCTRL_RESERVED_CLOSID);
 682}
 683
 684static int resctrl_arch_online_cpu(unsigned int cpu)
 685{
 686	struct rdt_resource *r;
 687
 688	mutex_lock(&domain_list_lock);
 689	for_each_capable_rdt_resource(r)
 690		domain_add_cpu(cpu, r);
 691	mutex_unlock(&domain_list_lock);
 692
 693	clear_closid_rmid(cpu);
 694	resctrl_online_cpu(cpu);
 695
 696	return 0;
 697}
 698
 699static int resctrl_arch_offline_cpu(unsigned int cpu)
 700{
 701	struct rdt_resource *r;
 702
 703	resctrl_offline_cpu(cpu);
 704
 705	mutex_lock(&domain_list_lock);
 706	for_each_capable_rdt_resource(r)
 707		domain_remove_cpu(cpu, r);
 708	mutex_unlock(&domain_list_lock);
 709
 710	clear_closid_rmid(cpu);
 711
 712	return 0;
 713}
 714
 715enum {
 716	RDT_FLAG_CMT,
 717	RDT_FLAG_MBM_TOTAL,
 718	RDT_FLAG_MBM_LOCAL,
 719	RDT_FLAG_L3_CAT,
 720	RDT_FLAG_L3_CDP,
 721	RDT_FLAG_L2_CAT,
 722	RDT_FLAG_L2_CDP,
 723	RDT_FLAG_MBA,
 724	RDT_FLAG_SMBA,
 725	RDT_FLAG_BMEC,
 726	RDT_FLAG_ABMC,
 727	RDT_FLAG_SDCIAE,
 728};
 729
 730#define RDT_OPT(idx, n, f)	\
 731[idx] = {			\
 732	.name = n,		\
 733	.flag = f		\
 734}
 735
 736struct rdt_options {
 737	char	*name;
 738	int	flag;
 739	bool	force_off, force_on;
 740};
 741
 742static struct rdt_options rdt_options[]  __ro_after_init = {
 743	RDT_OPT(RDT_FLAG_CMT,	    "cmt",	X86_FEATURE_CQM_OCCUP_LLC),
 744	RDT_OPT(RDT_FLAG_MBM_TOTAL, "mbmtotal", X86_FEATURE_CQM_MBM_TOTAL),
 745	RDT_OPT(RDT_FLAG_MBM_LOCAL, "mbmlocal", X86_FEATURE_CQM_MBM_LOCAL),
 746	RDT_OPT(RDT_FLAG_L3_CAT,    "l3cat",	X86_FEATURE_CAT_L3),
 747	RDT_OPT(RDT_FLAG_L3_CDP,    "l3cdp",	X86_FEATURE_CDP_L3),
 748	RDT_OPT(RDT_FLAG_L2_CAT,    "l2cat",	X86_FEATURE_CAT_L2),
 749	RDT_OPT(RDT_FLAG_L2_CDP,    "l2cdp",	X86_FEATURE_CDP_L2),
 750	RDT_OPT(RDT_FLAG_MBA,	    "mba",	X86_FEATURE_MBA),
 751	RDT_OPT(RDT_FLAG_SMBA,	    "smba",	X86_FEATURE_SMBA),
 752	RDT_OPT(RDT_FLAG_BMEC,	    "bmec",	X86_FEATURE_BMEC),
 753	RDT_OPT(RDT_FLAG_ABMC,	    "abmc",	X86_FEATURE_ABMC),
 754	RDT_OPT(RDT_FLAG_SDCIAE,    "sdciae",	X86_FEATURE_SDCIAE),
 755};
 756#define NUM_RDT_OPTIONS ARRAY_SIZE(rdt_options)
 757
 758static int __init set_rdt_options(char *str)
 759{
 760	struct rdt_options *o;
 761	bool force_off;
 762	char *tok;
 763
 764	if (*str == '=')
 765		str++;
 766	while ((tok = strsep(&str, ",")) != NULL) {
 767		force_off = *tok == '!';
 768		if (force_off)
 769			tok++;
 770		for (o = rdt_options; o < &rdt_options[NUM_RDT_OPTIONS]; o++) {
 771			if (strcmp(tok, o->name) == 0) {
 772				if (force_off)
 773					o->force_off = true;
 774				else
 775					o->force_on = true;
 776				break;
 777			}
 778		}
 779	}
 780	return 1;
 781}
 782__setup("rdt", set_rdt_options);
 783
 784bool rdt_cpu_has(int flag)
 785{
 786	bool ret = boot_cpu_has(flag);
 787	struct rdt_options *o;
 788
 789	if (!ret)
 790		return ret;
 791
 792	for (o = rdt_options; o < &rdt_options[NUM_RDT_OPTIONS]; o++) {
 793		if (flag == o->flag) {
 794			if (o->force_off)
 795				ret = false;
 796			if (o->force_on)
 797				ret = true;
 798			break;
 799		}
 800	}
 801	return ret;
 802}
 803
 804bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt)
 805{
 806	if (!rdt_cpu_has(X86_FEATURE_BMEC))
 807		return false;
 808
 809	switch (evt) {
 810	case QOS_L3_MBM_TOTAL_EVENT_ID:
 811		return rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL);
 812	case QOS_L3_MBM_LOCAL_EVENT_ID:
 813		return rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL);
 814	default:
 815		return false;
 816	}
 817}
 818
 819static __init bool get_mem_config(void)
 820{
 821	struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_MBA];
 822
 823	if (!rdt_cpu_has(X86_FEATURE_MBA))
 824		return false;
 825
 826	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
 827		return __get_mem_config_intel(&hw_res->r_resctrl);
 828	else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
 829		 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
 830		return __rdt_get_mem_config_amd(&hw_res->r_resctrl);
 831
 832	return false;
 833}
 834
 835static __init bool get_slow_mem_config(void)
 836{
 837	struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_SMBA];
 838
 839	if (!rdt_cpu_has(X86_FEATURE_SMBA))
 840		return false;
 841
 842	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
 843		return __rdt_get_mem_config_amd(&hw_res->r_resctrl);
 844
 845	return false;
 846}
 847
 848static __init bool get_rdt_alloc_resources(void)
 849{
 850	struct rdt_resource *r;
 851	bool ret = false;
 852
 853	if (rdt_alloc_capable)
 854		return true;
 855
 856	if (!boot_cpu_has(X86_FEATURE_RDT_A))
 857		return false;
 858
 859	if (rdt_cpu_has(X86_FEATURE_CAT_L3)) {
 860		r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
 861		rdt_get_cache_alloc_cfg(1, r);
 862		if (rdt_cpu_has(X86_FEATURE_CDP_L3))
 863			rdt_get_cdp_l3_config();
 864		if (rdt_cpu_has(X86_FEATURE_SDCIAE))
 865			rdt_set_io_alloc_capable(r);
 866		ret = true;
 867	}
 868	if (rdt_cpu_has(X86_FEATURE_CAT_L2)) {
 869		/* CPUID 0x10.2 fields are same format at 0x10.1 */
 870		r = &rdt_resources_all[RDT_RESOURCE_L2].r_resctrl;
 871		rdt_get_cache_alloc_cfg(2, r);
 872		if (rdt_cpu_has(X86_FEATURE_CDP_L2))
 873			rdt_get_cdp_l2_config();
 874		ret = true;
 875	}
 876
 877	if (get_mem_config())
 878		ret = true;
 879
 880	if (get_slow_mem_config())
 881		ret = true;
 882
 883	return ret;
 884}
 885
 886static __init bool get_rdt_mon_resources(void)
 887{
 888	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
 889	bool ret = false;
 890
 891	if (rdt_cpu_has(X86_FEATURE_CQM_OCCUP_LLC)) {
 892		resctrl_enable_mon_event(QOS_L3_OCCUP_EVENT_ID);
 893		ret = true;
 894	}
 895	if (rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL)) {
 896		resctrl_enable_mon_event(QOS_L3_MBM_TOTAL_EVENT_ID);
 897		ret = true;
 898	}
 899	if (rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL)) {
 900		resctrl_enable_mon_event(QOS_L3_MBM_LOCAL_EVENT_ID);
 901		ret = true;
 902	}
 903	if (rdt_cpu_has(X86_FEATURE_ABMC))
 904		ret = true;
 905
 906	if (!ret)
 907		return false;
 908
 909	return !rdt_get_mon_l3_config(r);
 910}
 911
 912static __init void __check_quirks_intel(void)
 913{
 914	switch (boot_cpu_data.x86_vfm) {
 915	case INTEL_HASWELL_X:
 916		if (!rdt_options[RDT_FLAG_L3_CAT].force_off)
 917			cache_alloc_hsw_probe();
 918		break;
 919	case INTEL_SKYLAKE_X:
 920		if (boot_cpu_data.x86_stepping <= 4)
 921			set_rdt_options("!cmt,!mbmtotal,!mbmlocal,!l3cat");
 922		else
 923			set_rdt_options("!l3cat");
 924		fallthrough;
 925	case INTEL_BROADWELL_X:
 926		intel_rdt_mbm_apply_quirk();
 927		break;
 928	}
 929}
 930
 931static __init void check_quirks(void)
 932{
 933	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
 934		__check_quirks_intel();
 935}
 936
 937static __init bool get_rdt_resources(void)
 938{
 939	rdt_alloc_capable = get_rdt_alloc_resources();
 940	rdt_mon_capable = get_rdt_mon_resources();
 941
 942	return (rdt_mon_capable || rdt_alloc_capable);
 943}
 944
 945static __init void rdt_init_res_defs_intel(void)
 946{
 947	struct rdt_hw_resource *hw_res;
 948	struct rdt_resource *r;
 949
 950	for_each_rdt_resource(r) {
 951		hw_res = resctrl_to_arch_res(r);
 952
 953		if (r->rid == RDT_RESOURCE_L3 ||
 954		    r->rid == RDT_RESOURCE_L2) {
 955			r->cache.arch_has_per_cpu_cfg = false;
 956			r->cache.min_cbm_bits = 1;
 957		} else if (r->rid == RDT_RESOURCE_MBA) {
 958			hw_res->msr_base = MSR_IA32_MBA_THRTL_BASE;
 959			hw_res->msr_update = mba_wrmsr_intel;
 960		}
 961	}
 962}
 963
 964static __init void rdt_init_res_defs_amd(void)
 965{
 966	struct rdt_hw_resource *hw_res;
 967	struct rdt_resource *r;
 968
 969	for_each_rdt_resource(r) {
 970		hw_res = resctrl_to_arch_res(r);
 971
 972		if (r->rid == RDT_RESOURCE_L3 ||
 973		    r->rid == RDT_RESOURCE_L2) {
 974			r->cache.arch_has_sparse_bitmasks = true;
 975			r->cache.arch_has_per_cpu_cfg = true;
 976			r->cache.min_cbm_bits = 0;
 977		} else if (r->rid == RDT_RESOURCE_MBA) {
 978			hw_res->msr_base = MSR_IA32_MBA_BW_BASE;
 979			hw_res->msr_update = mba_wrmsr_amd;
 980		} else if (r->rid == RDT_RESOURCE_SMBA) {
 981			hw_res->msr_base = MSR_IA32_SMBA_BW_BASE;
 982			hw_res->msr_update = mba_wrmsr_amd;
 983		}
 984	}
 985}
 986
 987static __init void rdt_init_res_defs(void)
 988{
 989	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
 990		rdt_init_res_defs_intel();
 991	else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
 992		 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
 993		rdt_init_res_defs_amd();
 994}
 995
 996static enum cpuhp_state rdt_online;
 997
 998/* Runs once on the BSP during boot. */
 999void resctrl_cpu_detect(struct cpuinfo_x86 *c)
1000{
1001	if (!cpu_has(c, X86_FEATURE_CQM_LLC) && !cpu_has(c, X86_FEATURE_ABMC)) {
1002		c->x86_cache_max_rmid  = -1;
1003		c->x86_cache_occ_scale = -1;
1004		c->x86_cache_mbm_width_offset = -1;
1005		return;
1006	}
1007
1008	/* will be overridden if occupancy monitoring exists */
1009	c->x86_cache_max_rmid = cpuid_ebx(0xf);
1010
1011	if (cpu_has(c, X86_FEATURE_CQM_OCCUP_LLC) ||
1012	    cpu_has(c, X86_FEATURE_CQM_MBM_TOTAL) ||
1013	    cpu_has(c, X86_FEATURE_CQM_MBM_LOCAL) ||
1014	    cpu_has(c, X86_FEATURE_ABMC)) {
1015		u32 eax, ebx, ecx, edx;
1016
1017		/* QoS sub-leaf, EAX=0Fh, ECX=1 */
1018		cpuid_count(0xf, 1, &eax, &ebx, &ecx, &edx);
1019
1020		c->x86_cache_max_rmid  = ecx;
1021		c->x86_cache_occ_scale = ebx;
1022		c->x86_cache_mbm_width_offset = eax & 0xff;
1023
1024		if (!c->x86_cache_mbm_width_offset) {
1025			switch (c->x86_vendor) {
1026			case X86_VENDOR_AMD:
1027				c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_AMD;
1028				break;
1029			case X86_VENDOR_HYGON:
1030				c->x86_cache_mbm_width_offset = MBM_CNTR_WIDTH_OFFSET_HYGON;
1031				break;
1032			default:
1033				/* Leave c->x86_cache_mbm_width_offset as 0 */
1034				break;
1035			}
1036		}
1037	}
1038}
1039
1040static int __init resctrl_arch_late_init(void)
1041{
1042	struct rdt_resource *r;
1043	int state, ret, i;
1044
1045	/* for_each_rdt_resource() requires all rid to be initialised. */
1046	for (i = 0; i < RDT_NUM_RESOURCES; i++)
1047		rdt_resources_all[i].r_resctrl.rid = i;
1048
1049	/*
1050	 * Initialize functions(or definitions) that are different
1051	 * between vendors here.
1052	 */
1053	rdt_init_res_defs();
1054
1055	check_quirks();
1056
1057	if (!get_rdt_resources())
1058		return -ENODEV;
1059
1060	state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
1061				  "x86/resctrl/cat:online:",
1062				  resctrl_arch_online_cpu,
1063				  resctrl_arch_offline_cpu);
1064	if (state < 0)
1065		return state;
1066
1067	ret = resctrl_init();
1068	if (ret) {
1069		cpuhp_remove_state(state);
1070		return ret;
1071	}
1072	rdt_online = state;
1073
1074	for_each_alloc_capable_rdt_resource(r)
1075		pr_info("%s allocation detected\n", r->name);
1076
1077	for_each_mon_capable_rdt_resource(r)
1078		pr_info("%s monitoring detected\n", r->name);
1079
1080	return 0;
1081}
1082
1083late_initcall(resctrl_arch_late_init);
1084
1085static void __exit resctrl_arch_exit(void)
1086{
1087	cpuhp_remove_state(rdt_online);
1088
1089	resctrl_exit();
1090}
1091
1092__exitcall(resctrl_arch_exit);