uncore.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2#include <linux/module.h>
   3
   4#include <asm/cpu_device_id.h>
   5#include <asm/intel-family.h>
   6#include <asm/msr.h>
   7#include "uncore.h"
   8#include "uncore_discovery.h"
   9
  10static bool uncore_no_discover;
  11module_param(uncore_no_discover, bool, 0);
  12MODULE_PARM_DESC(uncore_no_discover, "Don't enable the Intel uncore PerfMon discovery mechanism "
  13				     "(default: enable the discovery mechanism).");
  14struct intel_uncore_type *empty_uncore[] = { NULL, };
  15struct intel_uncore_type **uncore_msr_uncores = empty_uncore;
  16struct intel_uncore_type **uncore_pci_uncores = empty_uncore;
  17struct intel_uncore_type **uncore_mmio_uncores = empty_uncore;
  18
  19static bool pcidrv_registered;
  20struct pci_driver *uncore_pci_driver;
  21/* The PCI driver for the device which the uncore doesn't own. */
  22struct pci_driver *uncore_pci_sub_driver;
  23/* pci bus to socket mapping */
  24DEFINE_RAW_SPINLOCK(pci2phy_map_lock);
  25struct list_head pci2phy_map_head = LIST_HEAD_INIT(pci2phy_map_head);
  26struct pci_extra_dev *uncore_extra_pci_dev;
  27int __uncore_max_dies;
  28
  29/* mask of cpus that collect uncore events */
  30static cpumask_t uncore_cpu_mask;
  31
  32/* constraint for the fixed counter */
  33static struct event_constraint uncore_constraint_fixed =
  34	EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL);
  35struct event_constraint uncore_constraint_empty =
  36	EVENT_CONSTRAINT(0, 0, 0);
  37
  38MODULE_DESCRIPTION("Support for Intel uncore performance events");
  39MODULE_LICENSE("GPL");
  40
  41int uncore_pcibus_to_dieid(struct pci_bus *bus)
  42{
  43	struct pci2phy_map *map;
  44	int die_id = -1;
  45
  46	raw_spin_lock(&pci2phy_map_lock);
  47	list_for_each_entry(map, &pci2phy_map_head, list) {
  48		if (map->segment == pci_domain_nr(bus)) {
  49			die_id = map->pbus_to_dieid[bus->number];
  50			break;
  51		}
  52	}
  53	raw_spin_unlock(&pci2phy_map_lock);
  54
  55	return die_id;
  56}
  57
  58int uncore_die_to_segment(int die)
  59{
  60	struct pci_bus *bus = NULL;
  61
  62	/* Find first pci bus which attributes to specified die. */
  63	while ((bus = pci_find_next_bus(bus)) &&
  64	       (die != uncore_pcibus_to_dieid(bus)))
  65		;
  66
  67	return bus ? pci_domain_nr(bus) : -EINVAL;
  68}
  69
  70/* Note: This API can only be used when NUMA information is available. */
  71int uncore_device_to_die(struct pci_dev *dev)
  72{
  73	int node = pcibus_to_node(dev->bus);
  74	int cpu;
  75
  76	for_each_cpu(cpu, cpumask_of_pcibus(dev->bus)) {
  77		struct cpuinfo_x86 *c = &cpu_data(cpu);
  78
  79		if (c->initialized && cpu_to_node(cpu) == node)
  80			return c->topo.logical_die_id;
  81	}
  82
  83	return -1;
  84}
  85
  86static void uncore_free_pcibus_map(void)
  87{
  88	struct pci2phy_map *map, *tmp;
  89
  90	list_for_each_entry_safe(map, tmp, &pci2phy_map_head, list) {
  91		list_del(&map->list);
  92		kfree(map);
  93	}
  94}
  95
  96struct pci2phy_map *__find_pci2phy_map(int segment)
  97{
  98	struct pci2phy_map *map, *alloc = NULL;
  99	int i;
 100
 101	lockdep_assert_held(&pci2phy_map_lock);
 102
 103lookup:
 104	list_for_each_entry(map, &pci2phy_map_head, list) {
 105		if (map->segment == segment)
 106			goto end;
 107	}
 108
 109	if (!alloc) {
 110		raw_spin_unlock(&pci2phy_map_lock);
 111		alloc = kmalloc(sizeof(struct pci2phy_map), GFP_KERNEL);
 112		raw_spin_lock(&pci2phy_map_lock);
 113
 114		if (!alloc)
 115			return NULL;
 116
 117		goto lookup;
 118	}
 119
 120	map = alloc;
 121	alloc = NULL;
 122	map->segment = segment;
 123	for (i = 0; i < 256; i++)
 124		map->pbus_to_dieid[i] = -1;
 125	list_add_tail(&map->list, &pci2phy_map_head);
 126
 127end:
 128	kfree(alloc);
 129	return map;
 130}
 131
 132ssize_t uncore_event_show(struct device *dev,
 133			  struct device_attribute *attr, char *buf)
 134{
 135	struct uncore_event_desc *event =
 136		container_of(attr, struct uncore_event_desc, attr);
 137	return sprintf(buf, "%s", event->config);
 138}
 139
 140struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
 141{
 142	unsigned int dieid = topology_logical_die_id(cpu);
 143
 144	/*
 145	 * The unsigned check also catches the '-1' return value for non
 146	 * existent mappings in the topology map.
 147	 */
 148	return dieid < uncore_max_dies() ? pmu->boxes[dieid] : NULL;
 149}
 150
 151u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event)
 152{
 153	u64 count;
 154
 155	rdmsrq(event->hw.event_base, count);
 156
 157	return count;
 158}
 159
 160void uncore_mmio_exit_box(struct intel_uncore_box *box)
 161{
 162	if (box->io_addr)
 163		iounmap(box->io_addr);
 164}
 165
 166u64 uncore_mmio_read_counter(struct intel_uncore_box *box,
 167			     struct perf_event *event)
 168{
 169	if (!box->io_addr)
 170		return 0;
 171
 172	if (!uncore_mmio_is_valid_offset(box, event->hw.event_base))
 173		return 0;
 174
 175	return readq(box->io_addr + event->hw.event_base);
 176}
 177
 178/*
 179 * generic get constraint function for shared match/mask registers.
 180 */
 181struct event_constraint *
 182uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
 183{
 184	struct intel_uncore_extra_reg *er;
 185	struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 186	struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
 187	unsigned long flags;
 188	bool ok = false;
 189
 190	/*
 191	 * reg->alloc can be set due to existing state, so for fake box we
 192	 * need to ignore this, otherwise we might fail to allocate proper
 193	 * fake state for this extra reg constraint.
 194	 */
 195	if (reg1->idx == EXTRA_REG_NONE ||
 196	    (!uncore_box_is_fake(box) && reg1->alloc))
 197		return NULL;
 198
 199	er = &box->shared_regs[reg1->idx];
 200	raw_spin_lock_irqsave(&er->lock, flags);
 201	if (!atomic_read(&er->ref) ||
 202	    (er->config1 == reg1->config && er->config2 == reg2->config)) {
 203		atomic_inc(&er->ref);
 204		er->config1 = reg1->config;
 205		er->config2 = reg2->config;
 206		ok = true;
 207	}
 208	raw_spin_unlock_irqrestore(&er->lock, flags);
 209
 210	if (ok) {
 211		if (!uncore_box_is_fake(box))
 212			reg1->alloc = 1;
 213		return NULL;
 214	}
 215
 216	return &uncore_constraint_empty;
 217}
 218
 219void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
 220{
 221	struct intel_uncore_extra_reg *er;
 222	struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 223
 224	/*
 225	 * Only put constraint if extra reg was actually allocated. Also
 226	 * takes care of event which do not use an extra shared reg.
 227	 *
 228	 * Also, if this is a fake box we shouldn't touch any event state
 229	 * (reg->alloc) and we don't care about leaving inconsistent box
 230	 * state either since it will be thrown out.
 231	 */
 232	if (uncore_box_is_fake(box) || !reg1->alloc)
 233		return;
 234
 235	er = &box->shared_regs[reg1->idx];
 236	atomic_dec(&er->ref);
 237	reg1->alloc = 0;
 238}
 239
 240u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx)
 241{
 242	struct intel_uncore_extra_reg *er;
 243	unsigned long flags;
 244	u64 config;
 245
 246	er = &box->shared_regs[idx];
 247
 248	raw_spin_lock_irqsave(&er->lock, flags);
 249	config = er->config;
 250	raw_spin_unlock_irqrestore(&er->lock, flags);
 251
 252	return config;
 253}
 254
 255static void uncore_assign_hw_event(struct intel_uncore_box *box,
 256				   struct perf_event *event, int idx)
 257{
 258	struct hw_perf_event *hwc = &event->hw;
 259
 260	hwc->idx = idx;
 261	hwc->last_tag = ++box->tags[idx];
 262
 263	if (uncore_pmc_fixed(hwc->idx)) {
 264		hwc->event_base = uncore_fixed_ctr(box);
 265		hwc->config_base = uncore_fixed_ctl(box);
 266		return;
 267	}
 268
 269	if (intel_generic_uncore_assign_hw_event(event, box))
 270		return;
 271
 272	hwc->config_base = uncore_event_ctl(box, hwc->idx);
 273	hwc->event_base  = uncore_perf_ctr(box, hwc->idx);
 274}
 275
 276void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event)
 277{
 278	u64 prev_count, new_count, delta;
 279	int shift;
 280
 281	if (uncore_pmc_freerunning(event->hw.idx))
 282		shift = 64 - uncore_freerunning_bits(box, event);
 283	else if (uncore_pmc_fixed(event->hw.idx))
 284		shift = 64 - uncore_fixed_ctr_bits(box);
 285	else
 286		shift = 64 - uncore_perf_ctr_bits(box);
 287
 288	/* the hrtimer might modify the previous event value */
 289again:
 290	prev_count = local64_read(&event->hw.prev_count);
 291	new_count = uncore_read_counter(box, event);
 292	if (local64_xchg(&event->hw.prev_count, new_count) != prev_count)
 293		goto again;
 294
 295	delta = (new_count << shift) - (prev_count << shift);
 296	delta >>= shift;
 297
 298	local64_add(delta, &event->count);
 299}
 300
 301/*
 302 * The overflow interrupt is unavailable for SandyBridge-EP, is broken
 303 * for SandyBridge. So we use hrtimer to periodically poll the counter
 304 * to avoid overflow.
 305 */
 306static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
 307{
 308	struct intel_uncore_box *box;
 309	struct perf_event *event;
 310	int bit;
 311
 312	box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
 313	if (!box->n_active || box->cpu != smp_processor_id())
 314		return HRTIMER_NORESTART;
 315
 316	/*
 317	 * handle boxes with an active event list as opposed to active
 318	 * counters
 319	 */
 320	list_for_each_entry(event, &box->active_list, active_entry) {
 321		uncore_perf_event_update(box, event);
 322	}
 323
 324	for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
 325		uncore_perf_event_update(box, box->events[bit]);
 326
 327	hrtimer_forward_now(hrtimer, ns_to_ktime(box->hrtimer_duration));
 328	return HRTIMER_RESTART;
 329}
 330
 331void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
 332{
 333	hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration),
 334		      HRTIMER_MODE_REL_PINNED_HARD);
 335}
 336
 337void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
 338{
 339	hrtimer_cancel(&box->hrtimer);
 340}
 341
 342static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
 343{
 344	hrtimer_setup(&box->hrtimer, uncore_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
 345}
 346
 347static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type,
 348						 int node)
 349{
 350	int i, size, numshared = type->num_shared_regs ;
 351	struct intel_uncore_box *box;
 352
 353	size = sizeof(*box) + numshared * sizeof(struct intel_uncore_extra_reg);
 354
 355	box = kzalloc_node(size, GFP_KERNEL, node);
 356	if (!box)
 357		return NULL;
 358
 359	for (i = 0; i < numshared; i++)
 360		raw_spin_lock_init(&box->shared_regs[i].lock);
 361
 362	uncore_pmu_init_hrtimer(box);
 363	box->cpu = -1;
 364	box->dieid = -1;
 365
 366	/* set default hrtimer timeout */
 367	box->hrtimer_duration = UNCORE_PMU_HRTIMER_INTERVAL;
 368
 369	INIT_LIST_HEAD(&box->active_list);
 370
 371	return box;
 372}
 373
 374/*
 375 * Using uncore_pmu_event_init pmu event_init callback
 376 * as a detection point for uncore events.
 377 */
 378static int uncore_pmu_event_init(struct perf_event *event);
 379
 380static bool is_box_event(struct intel_uncore_box *box, struct perf_event *event)
 381{
 382	return &box->pmu->pmu == event->pmu;
 383}
 384
 385static int
 386uncore_collect_events(struct intel_uncore_box *box, struct perf_event *leader,
 387		      bool dogrp)
 388{
 389	struct perf_event *event;
 390	int n, max_count;
 391
 392	max_count = box->pmu->type->num_counters;
 393	if (box->pmu->type->fixed_ctl)
 394		max_count++;
 395
 396	if (box->n_events >= max_count)
 397		return -EINVAL;
 398
 399	n = box->n_events;
 400
 401	if (is_box_event(box, leader)) {
 402		box->event_list[n] = leader;
 403		n++;
 404	}
 405
 406	if (!dogrp)
 407		return n;
 408
 409	for_each_sibling_event(event, leader) {
 410		if (!is_box_event(box, event) ||
 411		    event->state <= PERF_EVENT_STATE_OFF)
 412			continue;
 413
 414		if (n >= max_count)
 415			return -EINVAL;
 416
 417		box->event_list[n] = event;
 418		n++;
 419	}
 420	return n;
 421}
 422
 423static struct event_constraint *
 424uncore_get_event_constraint(struct intel_uncore_box *box, struct perf_event *event)
 425{
 426	struct intel_uncore_type *type = box->pmu->type;
 427	struct event_constraint *c;
 428
 429	if (type->ops->get_constraint) {
 430		c = type->ops->get_constraint(box, event);
 431		if (c)
 432			return c;
 433	}
 434
 435	if (event->attr.config == UNCORE_FIXED_EVENT)
 436		return &uncore_constraint_fixed;
 437
 438	if (type->constraints) {
 439		for_each_event_constraint(c, type->constraints) {
 440			if ((event->hw.config & c->cmask) == c->code)
 441				return c;
 442		}
 443	}
 444
 445	return &type->unconstrainted;
 446}
 447
 448static void uncore_put_event_constraint(struct intel_uncore_box *box,
 449					struct perf_event *event)
 450{
 451	if (box->pmu->type->ops->put_constraint)
 452		box->pmu->type->ops->put_constraint(box, event);
 453}
 454
 455static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int n)
 456{
 457	unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
 458	struct event_constraint *c;
 459	int i, wmin, wmax, ret = 0;
 460	struct hw_perf_event *hwc;
 461
 462	bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX);
 463
 464	for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) {
 465		c = uncore_get_event_constraint(box, box->event_list[i]);
 466		box->event_constraint[i] = c;
 467		wmin = min(wmin, c->weight);
 468		wmax = max(wmax, c->weight);
 469	}
 470
 471	/* fastpath, try to reuse previous register */
 472	for (i = 0; i < n; i++) {
 473		hwc = &box->event_list[i]->hw;
 474		c = box->event_constraint[i];
 475
 476		/* never assigned */
 477		if (hwc->idx == -1)
 478			break;
 479
 480		/* constraint still honored */
 481		if (!test_bit(hwc->idx, c->idxmsk))
 482			break;
 483
 484		/* not already used */
 485		if (test_bit(hwc->idx, used_mask))
 486			break;
 487
 488		__set_bit(hwc->idx, used_mask);
 489		if (assign)
 490			assign[i] = hwc->idx;
 491	}
 492	/* slow path */
 493	if (i != n)
 494		ret = perf_assign_events(box->event_constraint, n,
 495					 wmin, wmax, n, assign);
 496
 497	if (!assign || ret) {
 498		for (i = 0; i < n; i++)
 499			uncore_put_event_constraint(box, box->event_list[i]);
 500	}
 501	return ret ? -EINVAL : 0;
 502}
 503
 504void uncore_pmu_event_start(struct perf_event *event, int flags)
 505{
 506	struct intel_uncore_box *box = uncore_event_to_box(event);
 507	int idx = event->hw.idx;
 508
 509	if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX))
 510		return;
 511
 512	/*
 513	 * Free running counter is read-only and always active.
 514	 * Use the current counter value as start point.
 515	 * There is no overflow interrupt for free running counter.
 516	 * Use hrtimer to periodically poll the counter to avoid overflow.
 517	 */
 518	if (uncore_pmc_freerunning(event->hw.idx)) {
 519		list_add_tail(&event->active_entry, &box->active_list);
 520		local64_set(&event->hw.prev_count,
 521			    uncore_read_counter(box, event));
 522		if (box->n_active++ == 0)
 523			uncore_pmu_start_hrtimer(box);
 524		return;
 525	}
 526
 527	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
 528		return;
 529
 530	event->hw.state = 0;
 531	box->events[idx] = event;
 532	box->n_active++;
 533	__set_bit(idx, box->active_mask);
 534
 535	local64_set(&event->hw.prev_count, uncore_read_counter(box, event));
 536	uncore_enable_event(box, event);
 537
 538	if (box->n_active == 1)
 539		uncore_pmu_start_hrtimer(box);
 540}
 541
 542void uncore_pmu_event_stop(struct perf_event *event, int flags)
 543{
 544	struct intel_uncore_box *box = uncore_event_to_box(event);
 545	struct hw_perf_event *hwc = &event->hw;
 546
 547	/* Cannot disable free running counter which is read-only */
 548	if (uncore_pmc_freerunning(hwc->idx)) {
 549		list_del(&event->active_entry);
 550		if (--box->n_active == 0)
 551			uncore_pmu_cancel_hrtimer(box);
 552		uncore_perf_event_update(box, event);
 553		return;
 554	}
 555
 556	if (__test_and_clear_bit(hwc->idx, box->active_mask)) {
 557		uncore_disable_event(box, event);
 558		box->n_active--;
 559		box->events[hwc->idx] = NULL;
 560		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
 561		hwc->state |= PERF_HES_STOPPED;
 562
 563		if (box->n_active == 0)
 564			uncore_pmu_cancel_hrtimer(box);
 565	}
 566
 567	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
 568		/*
 569		 * Drain the remaining delta count out of a event
 570		 * that we are disabling:
 571		 */
 572		uncore_perf_event_update(box, event);
 573		hwc->state |= PERF_HES_UPTODATE;
 574	}
 575}
 576
 577int uncore_pmu_event_add(struct perf_event *event, int flags)
 578{
 579	struct intel_uncore_box *box = uncore_event_to_box(event);
 580	struct hw_perf_event *hwc = &event->hw;
 581	int assign[UNCORE_PMC_IDX_MAX];
 582	int i, n, ret;
 583
 584	if (!box)
 585		return -ENODEV;
 586
 587	/*
 588	 * The free funning counter is assigned in event_init().
 589	 * The free running counter event and free running counter
 590	 * are 1:1 mapped. It doesn't need to be tracked in event_list.
 591	 */
 592	if (uncore_pmc_freerunning(hwc->idx)) {
 593		if (flags & PERF_EF_START)
 594			uncore_pmu_event_start(event, 0);
 595		return 0;
 596	}
 597
 598	ret = n = uncore_collect_events(box, event, false);
 599	if (ret < 0)
 600		return ret;
 601
 602	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 603	if (!(flags & PERF_EF_START))
 604		hwc->state |= PERF_HES_ARCH;
 605
 606	ret = uncore_assign_events(box, assign, n);
 607	if (ret)
 608		return ret;
 609
 610	/* save events moving to new counters */
 611	for (i = 0; i < box->n_events; i++) {
 612		event = box->event_list[i];
 613		hwc = &event->hw;
 614
 615		if (hwc->idx == assign[i] &&
 616			hwc->last_tag == box->tags[assign[i]])
 617			continue;
 618		/*
 619		 * Ensure we don't accidentally enable a stopped
 620		 * counter simply because we rescheduled.
 621		 */
 622		if (hwc->state & PERF_HES_STOPPED)
 623			hwc->state |= PERF_HES_ARCH;
 624
 625		uncore_pmu_event_stop(event, PERF_EF_UPDATE);
 626	}
 627
 628	/* reprogram moved events into new counters */
 629	for (i = 0; i < n; i++) {
 630		event = box->event_list[i];
 631		hwc = &event->hw;
 632
 633		if (hwc->idx != assign[i] ||
 634			hwc->last_tag != box->tags[assign[i]])
 635			uncore_assign_hw_event(box, event, assign[i]);
 636		else if (i < box->n_events)
 637			continue;
 638
 639		if (hwc->state & PERF_HES_ARCH)
 640			continue;
 641
 642		uncore_pmu_event_start(event, 0);
 643	}
 644	box->n_events = n;
 645
 646	return 0;
 647}
 648
 649void uncore_pmu_event_del(struct perf_event *event, int flags)
 650{
 651	struct intel_uncore_box *box = uncore_event_to_box(event);
 652	int i;
 653
 654	uncore_pmu_event_stop(event, PERF_EF_UPDATE);
 655
 656	/*
 657	 * The event for free running counter is not tracked by event_list.
 658	 * It doesn't need to force event->hw.idx = -1 to reassign the counter.
 659	 * Because the event and the free running counter are 1:1 mapped.
 660	 */
 661	if (uncore_pmc_freerunning(event->hw.idx))
 662		return;
 663
 664	for (i = 0; i < box->n_events; i++) {
 665		if (event == box->event_list[i]) {
 666			uncore_put_event_constraint(box, event);
 667
 668			for (++i; i < box->n_events; i++)
 669				box->event_list[i - 1] = box->event_list[i];
 670
 671			--box->n_events;
 672			break;
 673		}
 674	}
 675
 676	event->hw.idx = -1;
 677	event->hw.last_tag = ~0ULL;
 678}
 679
 680void uncore_pmu_event_read(struct perf_event *event)
 681{
 682	struct intel_uncore_box *box = uncore_event_to_box(event);
 683	uncore_perf_event_update(box, event);
 684}
 685
 686/*
 687 * validation ensures the group can be loaded onto the
 688 * PMU if it was the only group available.
 689 */
 690static int uncore_validate_group(struct intel_uncore_pmu *pmu,
 691				struct perf_event *event)
 692{
 693	struct perf_event *leader = event->group_leader;
 694	struct intel_uncore_box *fake_box;
 695	int ret = -EINVAL, n;
 696
 697	/* The free running counter is always active. */
 698	if (uncore_pmc_freerunning(event->hw.idx))
 699		return 0;
 700
 701	fake_box = uncore_alloc_box(pmu->type, NUMA_NO_NODE);
 702	if (!fake_box)
 703		return -ENOMEM;
 704
 705	fake_box->pmu = pmu;
 706	/*
 707	 * the event is not yet connected with its
 708	 * siblings therefore we must first collect
 709	 * existing siblings, then add the new event
 710	 * before we can simulate the scheduling
 711	 */
 712	n = uncore_collect_events(fake_box, leader, true);
 713	if (n < 0)
 714		goto out;
 715
 716	fake_box->n_events = n;
 717	n = uncore_collect_events(fake_box, event, false);
 718	if (n < 0)
 719		goto out;
 720
 721	fake_box->n_events = n;
 722
 723	ret = uncore_assign_events(fake_box, NULL, n);
 724out:
 725	kfree(fake_box);
 726	return ret;
 727}
 728
 729static int uncore_pmu_event_init(struct perf_event *event)
 730{
 731	struct intel_uncore_pmu *pmu;
 732	struct intel_uncore_box *box;
 733	struct hw_perf_event *hwc = &event->hw;
 734	int ret;
 735
 736	if (event->attr.type != event->pmu->type)
 737		return -ENOENT;
 738
 739	pmu = uncore_event_to_pmu(event);
 740	/* no device found for this pmu */
 741	if (!pmu->registered)
 742		return -ENOENT;
 743
 744	/* Sampling not supported yet */
 745	if (hwc->sample_period)
 746		return -EINVAL;
 747
 748	/*
 749	 * Place all uncore events for a particular physical package
 750	 * onto a single cpu
 751	 */
 752	if (event->cpu < 0)
 753		return -EINVAL;
 754	box = uncore_pmu_to_box(pmu, event->cpu);
 755	if (!box || box->cpu < 0)
 756		return -EINVAL;
 757	event->cpu = box->cpu;
 758	event->pmu_private = box;
 759
 760	event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
 761
 762	event->hw.idx = -1;
 763	event->hw.last_tag = ~0ULL;
 764	event->hw.extra_reg.idx = EXTRA_REG_NONE;
 765	event->hw.branch_reg.idx = EXTRA_REG_NONE;
 766
 767	if (event->attr.config == UNCORE_FIXED_EVENT) {
 768		/* no fixed counter */
 769		if (!pmu->type->fixed_ctl)
 770			return -EINVAL;
 771		/*
 772		 * if there is only one fixed counter, only the first pmu
 773		 * can access the fixed counter
 774		 */
 775		if (pmu->type->single_fixed && pmu->pmu_idx > 0)
 776			return -EINVAL;
 777
 778		/* fixed counters have event field hardcoded to zero */
 779		hwc->config = 0ULL;
 780	} else if (is_freerunning_event(event)) {
 781		hwc->config = event->attr.config;
 782		if (!check_valid_freerunning_event(box, event))
 783			return -EINVAL;
 784		event->hw.idx = UNCORE_PMC_IDX_FREERUNNING;
 785		/*
 786		 * The free running counter event and free running counter
 787		 * are always 1:1 mapped.
 788		 * The free running counter is always active.
 789		 * Assign the free running counter here.
 790		 */
 791		event->hw.event_base = uncore_freerunning_counter(box, event);
 792	} else {
 793		hwc->config = event->attr.config &
 794			      (pmu->type->event_mask | ((u64)pmu->type->event_mask_ext << 32));
 795		if (pmu->type->ops->hw_config) {
 796			ret = pmu->type->ops->hw_config(box, event);
 797			if (ret)
 798				return ret;
 799		}
 800	}
 801
 802	if (event->group_leader != event)
 803		ret = uncore_validate_group(pmu, event);
 804	else
 805		ret = 0;
 806
 807	return ret;
 808}
 809
 810static void uncore_pmu_enable(struct pmu *pmu)
 811{
 812	struct intel_uncore_pmu *uncore_pmu;
 813	struct intel_uncore_box *box;
 814
 815	uncore_pmu = container_of(pmu, struct intel_uncore_pmu, pmu);
 816
 817	box = uncore_pmu_to_box(uncore_pmu, smp_processor_id());
 818	if (!box)
 819		return;
 820
 821	if (uncore_pmu->type->ops->enable_box)
 822		uncore_pmu->type->ops->enable_box(box);
 823}
 824
 825static void uncore_pmu_disable(struct pmu *pmu)
 826{
 827	struct intel_uncore_pmu *uncore_pmu;
 828	struct intel_uncore_box *box;
 829
 830	uncore_pmu = container_of(pmu, struct intel_uncore_pmu, pmu);
 831
 832	box = uncore_pmu_to_box(uncore_pmu, smp_processor_id());
 833	if (!box)
 834		return;
 835
 836	if (uncore_pmu->type->ops->disable_box)
 837		uncore_pmu->type->ops->disable_box(box);
 838}
 839
 840static ssize_t uncore_get_attr_cpumask(struct device *dev,
 841				struct device_attribute *attr, char *buf)
 842{
 843	struct intel_uncore_pmu *pmu = container_of(dev_get_drvdata(dev), struct intel_uncore_pmu, pmu);
 844
 845	return cpumap_print_to_pagebuf(true, buf, &pmu->cpu_mask);
 846}
 847
 848static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL);
 849
 850static struct attribute *uncore_pmu_attrs[] = {
 851	&dev_attr_cpumask.attr,
 852	NULL,
 853};
 854
 855static const struct attribute_group uncore_pmu_attr_group = {
 856	.attrs = uncore_pmu_attrs,
 857};
 858
 859static inline int uncore_get_box_id(struct intel_uncore_type *type,
 860				    struct intel_uncore_pmu *pmu)
 861{
 862	if (type->boxes)
 863		return intel_uncore_find_discovery_unit_id(type->boxes, -1, pmu->pmu_idx);
 864
 865	return pmu->pmu_idx;
 866}
 867
 868void uncore_get_alias_name(char *pmu_name, struct intel_uncore_pmu *pmu)
 869{
 870	struct intel_uncore_type *type = pmu->type;
 871
 872	if (type->num_boxes == 1)
 873		sprintf(pmu_name, "uncore_type_%u", type->type_id);
 874	else {
 875		sprintf(pmu_name, "uncore_type_%u_%d",
 876			type->type_id, uncore_get_box_id(type, pmu));
 877	}
 878}
 879
 880static void uncore_get_pmu_name(struct intel_uncore_pmu *pmu)
 881{
 882	struct intel_uncore_type *type = pmu->type;
 883
 884	/*
 885	 * No uncore block name in discovery table.
 886	 * Use uncore_type_&typeid_&boxid as name.
 887	 */
 888	if (!type->name) {
 889		uncore_get_alias_name(pmu->name, pmu);
 890		return;
 891	}
 892
 893	if (type->num_boxes == 1) {
 894		if (strlen(type->name) > 0)
 895			sprintf(pmu->name, "uncore_%s", type->name);
 896		else
 897			sprintf(pmu->name, "uncore");
 898	} else {
 899		/*
 900		 * Use the box ID from the discovery table if applicable.
 901		 */
 902		sprintf(pmu->name, "uncore_%s_%d", type->name,
 903			uncore_get_box_id(type, pmu));
 904	}
 905}
 906
 907static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
 908{
 909	int ret;
 910
 911	if (!pmu->type->pmu) {
 912		pmu->pmu = (struct pmu) {
 913			.attr_groups	= pmu->type->attr_groups,
 914			.task_ctx_nr	= perf_invalid_context,
 915			.pmu_enable	= uncore_pmu_enable,
 916			.pmu_disable	= uncore_pmu_disable,
 917			.event_init	= uncore_pmu_event_init,
 918			.add		= uncore_pmu_event_add,
 919			.del		= uncore_pmu_event_del,
 920			.start		= uncore_pmu_event_start,
 921			.stop		= uncore_pmu_event_stop,
 922			.read		= uncore_pmu_event_read,
 923			.module		= THIS_MODULE,
 924			.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
 925			.attr_update	= pmu->type->attr_update,
 926		};
 927	} else {
 928		pmu->pmu = *pmu->type->pmu;
 929		pmu->pmu.attr_groups = pmu->type->attr_groups;
 930		pmu->pmu.attr_update = pmu->type->attr_update;
 931	}
 932
 933	uncore_get_pmu_name(pmu);
 934
 935	ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
 936	if (!ret)
 937		pmu->registered = true;
 938	return ret;
 939}
 940
 941static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu)
 942{
 943	if (!pmu->registered)
 944		return;
 945	perf_pmu_unregister(&pmu->pmu);
 946	pmu->registered = false;
 947}
 948
 949static void uncore_free_boxes(struct intel_uncore_pmu *pmu)
 950{
 951	int die;
 952
 953	for (die = 0; die < uncore_max_dies(); die++)
 954		kfree(pmu->boxes[die]);
 955	kfree(pmu->boxes);
 956}
 957
 958static void uncore_type_exit(struct intel_uncore_type *type)
 959{
 960	struct intel_uncore_pmu *pmu = type->pmus;
 961	int i;
 962
 963	if (type->cleanup_mapping)
 964		type->cleanup_mapping(type);
 965
 966	if (type->cleanup_extra_boxes)
 967		type->cleanup_extra_boxes(type);
 968
 969	if (pmu) {
 970		for (i = 0; i < type->num_boxes; i++, pmu++) {
 971			uncore_pmu_unregister(pmu);
 972			uncore_free_boxes(pmu);
 973		}
 974		kfree(type->pmus);
 975		type->pmus = NULL;
 976	}
 977
 978	kfree(type->events_group);
 979	type->events_group = NULL;
 980}
 981
 982static void uncore_types_exit(struct intel_uncore_type **types)
 983{
 984	for (; *types; types++)
 985		uncore_type_exit(*types);
 986}
 987
 988static int __init uncore_type_init(struct intel_uncore_type *type)
 989{
 990	struct intel_uncore_pmu *pmus;
 991	size_t size;
 992	int i, j;
 993
 994	pmus = kcalloc(type->num_boxes, sizeof(*pmus), GFP_KERNEL);
 995	if (!pmus)
 996		return -ENOMEM;
 997
 998	size = uncore_max_dies() * sizeof(struct intel_uncore_box *);
 999
1000	for (i = 0; i < type->num_boxes; i++) {
1001		pmus[i].pmu_idx	= i;
1002		pmus[i].type	= type;
1003		pmus[i].boxes	= kzalloc(size, GFP_KERNEL);
1004		if (!pmus[i].boxes)
1005			goto err;
1006	}
1007
1008	type->pmus = pmus;
1009	type->unconstrainted = (struct event_constraint)
1010		__EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
1011				0, type->num_counters, 0, 0);
1012
1013	if (type->event_descs) {
1014		struct {
1015			struct attribute_group group;
1016			struct attribute *attrs[];
1017		} *attr_group;
1018		for (i = 0; type->event_descs[i].attr.attr.name; i++);
1019
1020		attr_group = kzalloc(struct_size(attr_group, attrs, i + 1),
1021								GFP_KERNEL);
1022		if (!attr_group)
1023			goto err;
1024
1025		attr_group->group.name = "events";
1026		attr_group->group.attrs = attr_group->attrs;
1027
1028		for (j = 0; j < i; j++)
1029			attr_group->attrs[j] = &type->event_descs[j].attr.attr;
1030
1031		type->events_group = &attr_group->group;
1032	}
1033
1034	type->pmu_group = &uncore_pmu_attr_group;
1035
1036	if (type->set_mapping)
1037		type->set_mapping(type);
1038
1039	return 0;
1040
1041err:
1042	for (i = 0; i < type->num_boxes; i++)
1043		kfree(pmus[i].boxes);
1044	kfree(pmus);
1045
1046	return -ENOMEM;
1047}
1048
1049static int __init
1050uncore_types_init(struct intel_uncore_type **types)
1051{
1052	int ret;
1053
1054	for (; *types; types++) {
1055		ret = uncore_type_init(*types);
1056		if (ret)
1057			return ret;
1058	}
1059	return 0;
1060}
1061
1062/*
1063 * Get the die information of a PCI device.
1064 * @pdev: The PCI device.
1065 * @die: The die id which the device maps to.
1066 */
1067static int uncore_pci_get_dev_die_info(struct pci_dev *pdev, int *die)
1068{
1069	*die = uncore_pcibus_to_dieid(pdev->bus);
1070	if (*die < 0)
1071		return -EINVAL;
1072
1073	return 0;
1074}
1075
1076static struct intel_uncore_pmu *
1077uncore_pci_find_dev_pmu_from_types(struct pci_dev *pdev)
1078{
1079	struct intel_uncore_type **types = uncore_pci_uncores;
1080	struct intel_uncore_discovery_unit *unit;
1081	struct intel_uncore_type *type;
1082	struct rb_node *node;
1083
1084	for (; *types; types++) {
1085		type = *types;
1086
1087		for (node = rb_first(type->boxes); node; node = rb_next(node)) {
1088			unit = rb_entry(node, struct intel_uncore_discovery_unit, node);
1089			if (pdev->devfn == UNCORE_DISCOVERY_PCI_DEVFN(unit->addr) &&
1090			    pdev->bus->number == UNCORE_DISCOVERY_PCI_BUS(unit->addr) &&
1091			    pci_domain_nr(pdev->bus) == UNCORE_DISCOVERY_PCI_DOMAIN(unit->addr))
1092				return &type->pmus[unit->pmu_idx];
1093		}
1094	}
1095
1096	return NULL;
1097}
1098
1099/*
1100 * Find the PMU of a PCI device.
1101 * @pdev: The PCI device.
1102 * @ids: The ID table of the available PCI devices with a PMU.
1103 *       If NULL, search the whole uncore_pci_uncores.
1104 */
1105static struct intel_uncore_pmu *
1106uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids)
1107{
1108	struct intel_uncore_pmu *pmu = NULL;
1109	struct intel_uncore_type *type;
1110	kernel_ulong_t data;
1111	unsigned int devfn;
1112
1113	if (!ids)
1114		return uncore_pci_find_dev_pmu_from_types(pdev);
1115
1116	while (ids && ids->vendor) {
1117		if ((ids->vendor == pdev->vendor) &&
1118		    (ids->device == pdev->device)) {
1119			data = ids->driver_data;
1120			devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(data),
1121					  UNCORE_PCI_DEV_FUNC(data));
1122			if (devfn == pdev->devfn) {
1123				type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(data)];
1124				pmu = &type->pmus[UNCORE_PCI_DEV_IDX(data)];
1125				break;
1126			}
1127		}
1128		ids++;
1129	}
1130	return pmu;
1131}
1132
1133/*
1134 * Register the PMU for a PCI device
1135 * @pdev: The PCI device.
1136 * @type: The corresponding PMU type of the device.
1137 * @pmu: The corresponding PMU of the device.
1138 * @die: The die id which the device maps to.
1139 */
1140static int uncore_pci_pmu_register(struct pci_dev *pdev,
1141				   struct intel_uncore_type *type,
1142				   struct intel_uncore_pmu *pmu,
1143				   int die)
1144{
1145	struct intel_uncore_box *box;
1146	int ret;
1147
1148	if (WARN_ON_ONCE(pmu->boxes[die] != NULL))
1149		return -EINVAL;
1150
1151	box = uncore_alloc_box(type, NUMA_NO_NODE);
1152	if (!box)
1153		return -ENOMEM;
1154
1155	atomic_inc(&box->refcnt);
1156	box->dieid = die;
1157	box->pci_dev = pdev;
1158	box->pmu = pmu;
1159	uncore_box_init(box);
1160
1161	pmu->boxes[die] = box;
1162	if (atomic_inc_return(&pmu->activeboxes) > 1)
1163		return 0;
1164
1165	/* First active box registers the pmu */
1166	ret = uncore_pmu_register(pmu);
1167	if (ret) {
1168		pmu->boxes[die] = NULL;
1169		uncore_box_exit(box);
1170		kfree(box);
1171	}
1172	return ret;
1173}
1174
1175/*
1176 * add a pci uncore device
1177 */
1178static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1179{
1180	struct intel_uncore_type *type;
1181	struct intel_uncore_pmu *pmu = NULL;
1182	int die, ret;
1183
1184	ret = uncore_pci_get_dev_die_info(pdev, &die);
1185	if (ret)
1186		return ret;
1187
1188	if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) {
1189		int idx = UNCORE_PCI_DEV_IDX(id->driver_data);
1190
1191		uncore_extra_pci_dev[die].dev[idx] = pdev;
1192		pci_set_drvdata(pdev, NULL);
1193		return 0;
1194	}
1195
1196	type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)];
1197
1198	/*
1199	 * Some platforms, e.g.  Knights Landing, use a common PCI device ID
1200	 * for multiple instances of an uncore PMU device type. We should check
1201	 * PCI slot and func to indicate the uncore box.
1202	 */
1203	if (id->driver_data & ~0xffff) {
1204		struct pci_driver *pci_drv = to_pci_driver(pdev->dev.driver);
1205
1206		pmu = uncore_pci_find_dev_pmu(pdev, pci_drv->id_table);
1207		if (pmu == NULL)
1208			return -ENODEV;
1209	} else {
1210		/*
1211		 * for performance monitoring unit with multiple boxes,
1212		 * each box has a different function id.
1213		 */
1214		pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)];
1215	}
1216
1217	ret = uncore_pci_pmu_register(pdev, type, pmu, die);
1218
1219	pci_set_drvdata(pdev, pmu->boxes[die]);
1220
1221	return ret;
1222}
1223
1224/*
1225 * Unregister the PMU of a PCI device
1226 * @pmu: The corresponding PMU is unregistered.
1227 * @die: The die id which the device maps to.
1228 */
1229static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die)
1230{
1231	struct intel_uncore_box *box = pmu->boxes[die];
1232
1233	pmu->boxes[die] = NULL;
1234	if (atomic_dec_return(&pmu->activeboxes) == 0)
1235		uncore_pmu_unregister(pmu);
1236	uncore_box_exit(box);
1237	kfree(box);
1238}
1239
1240static void uncore_pci_remove(struct pci_dev *pdev)
1241{
1242	struct intel_uncore_box *box;
1243	struct intel_uncore_pmu *pmu;
1244	int i, die;
1245
1246	if (uncore_pci_get_dev_die_info(pdev, &die))
1247		return;
1248
1249	box = pci_get_drvdata(pdev);
1250	if (!box) {
1251		for (i = 0; i < UNCORE_EXTRA_PCI_DEV_MAX; i++) {
1252			if (uncore_extra_pci_dev[die].dev[i] == pdev) {
1253				uncore_extra_pci_dev[die].dev[i] = NULL;
1254				break;
1255			}
1256		}
1257		WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX);
1258		return;
1259	}
1260
1261	pmu = box->pmu;
1262
1263	pci_set_drvdata(pdev, NULL);
1264
1265	uncore_pci_pmu_unregister(pmu, die);
1266}
1267
1268static int uncore_bus_notify(struct notifier_block *nb,
1269			     unsigned long action, void *data,
1270			     const struct pci_device_id *ids)
1271{
1272	struct device *dev = data;
1273	struct pci_dev *pdev = to_pci_dev(dev);
1274	struct intel_uncore_pmu *pmu;
1275	int die;
1276
1277	/* Unregister the PMU when the device is going to be deleted. */
1278	if (action != BUS_NOTIFY_DEL_DEVICE)
1279		return NOTIFY_DONE;
1280
1281	pmu = uncore_pci_find_dev_pmu(pdev, ids);
1282	if (!pmu)
1283		return NOTIFY_DONE;
1284
1285	if (uncore_pci_get_dev_die_info(pdev, &die))
1286		return NOTIFY_DONE;
1287
1288	uncore_pci_pmu_unregister(pmu, die);
1289
1290	return NOTIFY_OK;
1291}
1292
1293static int uncore_pci_sub_bus_notify(struct notifier_block *nb,
1294				     unsigned long action, void *data)
1295{
1296	return uncore_bus_notify(nb, action, data,
1297				 uncore_pci_sub_driver->id_table);
1298}
1299
1300static struct notifier_block uncore_pci_sub_notifier = {
1301	.notifier_call = uncore_pci_sub_bus_notify,
1302};
1303
1304static void uncore_pci_sub_driver_init(void)
1305{
1306	const struct pci_device_id *ids = uncore_pci_sub_driver->id_table;
1307	struct intel_uncore_type *type;
1308	struct intel_uncore_pmu *pmu;
1309	struct pci_dev *pci_sub_dev;
1310	bool notify = false;
1311	unsigned int devfn;
1312	int die;
1313
1314	while (ids && ids->vendor) {
1315		pci_sub_dev = NULL;
1316		type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(ids->driver_data)];
1317		/*
1318		 * Search the available device, and register the
1319		 * corresponding PMU.
1320		 */
1321		while ((pci_sub_dev = pci_get_device(PCI_VENDOR_ID_INTEL,
1322						     ids->device, pci_sub_dev))) {
1323			devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(ids->driver_data),
1324					  UNCORE_PCI_DEV_FUNC(ids->driver_data));
1325			if (devfn != pci_sub_dev->devfn)
1326				continue;
1327
1328			pmu = &type->pmus[UNCORE_PCI_DEV_IDX(ids->driver_data)];
1329
1330			if (uncore_pci_get_dev_die_info(pci_sub_dev, &die))
1331				continue;
1332
1333			if (!uncore_pci_pmu_register(pci_sub_dev, type, pmu,
1334						     die))
1335				notify = true;
1336		}
1337		ids++;
1338	}
1339
1340	if (notify && bus_register_notifier(&pci_bus_type, &uncore_pci_sub_notifier))
1341		notify = false;
1342
1343	if (!notify)
1344		uncore_pci_sub_driver = NULL;
1345}
1346
1347static int uncore_pci_bus_notify(struct notifier_block *nb,
1348				     unsigned long action, void *data)
1349{
1350	return uncore_bus_notify(nb, action, data, NULL);
1351}
1352
1353static struct notifier_block uncore_pci_notifier = {
1354	.notifier_call = uncore_pci_bus_notify,
1355};
1356
1357
1358static void uncore_pci_pmus_register(void)
1359{
1360	struct intel_uncore_type **types = uncore_pci_uncores;
1361	struct intel_uncore_discovery_unit *unit;
1362	struct intel_uncore_type *type;
1363	struct intel_uncore_pmu *pmu;
1364	struct rb_node *node;
1365	struct pci_dev *pdev;
1366
1367	for (; *types; types++) {
1368		type = *types;
1369
1370		for (node = rb_first(type->boxes); node; node = rb_next(node)) {
1371			unit = rb_entry(node, struct intel_uncore_discovery_unit, node);
1372			pdev = pci_get_domain_bus_and_slot(UNCORE_DISCOVERY_PCI_DOMAIN(unit->addr),
1373							   UNCORE_DISCOVERY_PCI_BUS(unit->addr),
1374							   UNCORE_DISCOVERY_PCI_DEVFN(unit->addr));
1375
1376			if (!pdev)
1377				continue;
1378			pmu = &type->pmus[unit->pmu_idx];
1379			uncore_pci_pmu_register(pdev, type, pmu, unit->die);
1380		}
1381	}
1382
1383	bus_register_notifier(&pci_bus_type, &uncore_pci_notifier);
1384}
1385
1386static int __init uncore_pci_init(void)
1387{
1388	size_t size;
1389	int ret;
1390
1391	size = uncore_max_dies() * sizeof(struct pci_extra_dev);
1392	uncore_extra_pci_dev = kzalloc(size, GFP_KERNEL);
1393	if (!uncore_extra_pci_dev) {
1394		ret = -ENOMEM;
1395		goto err;
1396	}
1397
1398	ret = uncore_types_init(uncore_pci_uncores);
1399	if (ret)
1400		goto errtype;
1401
1402	if (uncore_pci_driver) {
1403		uncore_pci_driver->probe = uncore_pci_probe;
1404		uncore_pci_driver->remove = uncore_pci_remove;
1405
1406		ret = pci_register_driver(uncore_pci_driver);
1407		if (ret)
1408			goto errtype;
1409	} else
1410		uncore_pci_pmus_register();
1411
1412	if (uncore_pci_sub_driver)
1413		uncore_pci_sub_driver_init();
1414
1415	pcidrv_registered = true;
1416	return 0;
1417
1418errtype:
1419	uncore_types_exit(uncore_pci_uncores);
1420	kfree(uncore_extra_pci_dev);
1421	uncore_extra_pci_dev = NULL;
1422	uncore_free_pcibus_map();
1423err:
1424	uncore_pci_uncores = empty_uncore;
1425	return ret;
1426}
1427
1428static void uncore_pci_exit(void)
1429{
1430	if (pcidrv_registered) {
1431		pcidrv_registered = false;
1432		if (uncore_pci_sub_driver)
1433			bus_unregister_notifier(&pci_bus_type, &uncore_pci_sub_notifier);
1434		if (uncore_pci_driver)
1435			pci_unregister_driver(uncore_pci_driver);
1436		else
1437			bus_unregister_notifier(&pci_bus_type, &uncore_pci_notifier);
1438		uncore_types_exit(uncore_pci_uncores);
1439		kfree(uncore_extra_pci_dev);
1440		uncore_free_pcibus_map();
1441	}
1442}
1443
1444static bool uncore_die_has_box(struct intel_uncore_type *type,
1445			       int die, unsigned int pmu_idx)
1446{
1447	if (!type->boxes)
1448		return true;
1449
1450	if (intel_uncore_find_discovery_unit_id(type->boxes, die, pmu_idx) < 0)
1451		return false;
1452
1453	return true;
1454}
1455
1456static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu,
1457				   int new_cpu)
1458{
1459	struct intel_uncore_pmu *pmu = type->pmus;
1460	struct intel_uncore_box *box;
1461	int i, die;
1462
1463	die = topology_logical_die_id(old_cpu < 0 ? new_cpu : old_cpu);
1464	for (i = 0; i < type->num_boxes; i++, pmu++) {
1465		box = pmu->boxes[die];
1466		if (!box)
1467			continue;
1468
1469		if (old_cpu < 0) {
1470			WARN_ON_ONCE(box->cpu != -1);
1471			if (uncore_die_has_box(type, die, pmu->pmu_idx)) {
1472				box->cpu = new_cpu;
1473				cpumask_set_cpu(new_cpu, &pmu->cpu_mask);
1474			}
1475			continue;
1476		}
1477
1478		WARN_ON_ONCE(box->cpu != -1 && box->cpu != old_cpu);
1479		box->cpu = -1;
1480		cpumask_clear_cpu(old_cpu, &pmu->cpu_mask);
1481		if (new_cpu < 0)
1482			continue;
1483
1484		if (!uncore_die_has_box(type, die, pmu->pmu_idx))
1485			continue;
1486		uncore_pmu_cancel_hrtimer(box);
1487		perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu);
1488		box->cpu = new_cpu;
1489		cpumask_set_cpu(new_cpu, &pmu->cpu_mask);
1490	}
1491}
1492
1493static void uncore_change_context(struct intel_uncore_type **uncores,
1494				  int old_cpu, int new_cpu)
1495{
1496	for (; *uncores; uncores++)
1497		uncore_change_type_ctx(*uncores, old_cpu, new_cpu);
1498}
1499
1500static void uncore_box_unref(struct intel_uncore_type **types, int id)
1501{
1502	struct intel_uncore_type *type;
1503	struct intel_uncore_pmu *pmu;
1504	struct intel_uncore_box *box;
1505	int i;
1506
1507	for (; *types; types++) {
1508		type = *types;
1509		pmu = type->pmus;
1510		for (i = 0; i < type->num_boxes; i++, pmu++) {
1511			box = pmu->boxes[id];
1512			if (box && box->cpu >= 0 && atomic_dec_return(&box->refcnt) == 0)
1513				uncore_box_exit(box);
1514		}
1515	}
1516}
1517
1518static int uncore_event_cpu_offline(unsigned int cpu)
1519{
1520	int die, target;
1521
1522	/* Check if exiting cpu is used for collecting uncore events */
1523	if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
1524		goto unref;
1525	/* Find a new cpu to collect uncore events */
1526	target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
1527
1528	/* Migrate uncore events to the new target */
1529	if (target < nr_cpu_ids)
1530		cpumask_set_cpu(target, &uncore_cpu_mask);
1531	else
1532		target = -1;
1533
1534	uncore_change_context(uncore_msr_uncores, cpu, target);
1535	uncore_change_context(uncore_mmio_uncores, cpu, target);
1536	uncore_change_context(uncore_pci_uncores, cpu, target);
1537
1538unref:
1539	/* Clear the references */
1540	die = topology_logical_die_id(cpu);
1541	uncore_box_unref(uncore_msr_uncores, die);
1542	uncore_box_unref(uncore_mmio_uncores, die);
1543	return 0;
1544}
1545
1546static int allocate_boxes(struct intel_uncore_type **types,
1547			 unsigned int die, unsigned int cpu)
1548{
1549	struct intel_uncore_box *box, *tmp;
1550	struct intel_uncore_type *type;
1551	struct intel_uncore_pmu *pmu;
1552	LIST_HEAD(allocated);
1553	int i;
1554
1555	/* Try to allocate all required boxes */
1556	for (; *types; types++) {
1557		type = *types;
1558		pmu = type->pmus;
1559		for (i = 0; i < type->num_boxes; i++, pmu++) {
1560			if (pmu->boxes[die])
1561				continue;
1562			box = uncore_alloc_box(type, cpu_to_node(cpu));
1563			if (!box)
1564				goto cleanup;
1565			box->pmu = pmu;
1566			box->dieid = die;
1567			list_add(&box->active_list, &allocated);
1568		}
1569	}
1570	/* Install them in the pmus */
1571	list_for_each_entry_safe(box, tmp, &allocated, active_list) {
1572		list_del_init(&box->active_list);
1573		box->pmu->boxes[die] = box;
1574	}
1575	return 0;
1576
1577cleanup:
1578	list_for_each_entry_safe(box, tmp, &allocated, active_list) {
1579		list_del_init(&box->active_list);
1580		kfree(box);
1581	}
1582	return -ENOMEM;
1583}
1584
1585static int uncore_box_ref(struct intel_uncore_type **types,
1586			  int id, unsigned int cpu)
1587{
1588	struct intel_uncore_type *type;
1589	struct intel_uncore_pmu *pmu;
1590	struct intel_uncore_box *box;
1591	int i, ret;
1592
1593	ret = allocate_boxes(types, id, cpu);
1594	if (ret)
1595		return ret;
1596
1597	for (; *types; types++) {
1598		type = *types;
1599		pmu = type->pmus;
1600		for (i = 0; i < type->num_boxes; i++, pmu++) {
1601			box = pmu->boxes[id];
1602			if (box && box->cpu >= 0 && atomic_inc_return(&box->refcnt) == 1)
1603				uncore_box_init(box);
1604		}
1605	}
1606	return 0;
1607}
1608
1609static int uncore_event_cpu_online(unsigned int cpu)
1610{
1611	int die, target, msr_ret, mmio_ret;
1612
1613	die = topology_logical_die_id(cpu);
1614	msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu);
1615	mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu);
1616	if (msr_ret && mmio_ret)
1617		return -ENOMEM;
1618
1619	/*
1620	 * Check if there is an online cpu in the package
1621	 * which collects uncore events already.
1622	 */
1623	target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
1624	if (target < nr_cpu_ids)
1625		return 0;
1626
1627	cpumask_set_cpu(cpu, &uncore_cpu_mask);
1628
1629	if (!msr_ret)
1630		uncore_change_context(uncore_msr_uncores, -1, cpu);
1631	if (!mmio_ret)
1632		uncore_change_context(uncore_mmio_uncores, -1, cpu);
1633	uncore_change_context(uncore_pci_uncores, -1, cpu);
1634	return 0;
1635}
1636
1637static int __init type_pmu_register(struct intel_uncore_type *type)
1638{
1639	int i, ret;
1640
1641	for (i = 0; i < type->num_boxes; i++) {
1642		ret = uncore_pmu_register(&type->pmus[i]);
1643		if (ret)
1644			return ret;
1645	}
1646	return 0;
1647}
1648
1649static int __init uncore_msr_pmus_register(void)
1650{
1651	struct intel_uncore_type **types = uncore_msr_uncores;
1652	int ret;
1653
1654	for (; *types; types++) {
1655		ret = type_pmu_register(*types);
1656		if (ret)
1657			return ret;
1658	}
1659	return 0;
1660}
1661
1662static int __init uncore_cpu_init(void)
1663{
1664	int ret;
1665
1666	ret = uncore_types_init(uncore_msr_uncores);
1667	if (ret)
1668		goto err;
1669
1670	ret = uncore_msr_pmus_register();
1671	if (ret)
1672		goto err;
1673	return 0;
1674err:
1675	uncore_types_exit(uncore_msr_uncores);
1676	uncore_msr_uncores = empty_uncore;
1677	return ret;
1678}
1679
1680static int __init uncore_mmio_init(void)
1681{
1682	struct intel_uncore_type **types = uncore_mmio_uncores;
1683	int ret;
1684
1685	ret = uncore_types_init(types);
1686	if (ret)
1687		goto err;
1688
1689	for (; *types; types++) {
1690		ret = type_pmu_register(*types);
1691		if (ret)
1692			goto err;
1693	}
1694	return 0;
1695err:
1696	uncore_types_exit(uncore_mmio_uncores);
1697	uncore_mmio_uncores = empty_uncore;
1698	return ret;
1699}
1700
1701struct intel_uncore_init_fun {
1702	void	(*cpu_init)(void);
1703	int	(*pci_init)(void);
1704	void	(*mmio_init)(void);
1705	/* Discovery table is required */
1706	bool	use_discovery;
1707	/* The units in the discovery table should be ignored. */
1708	int	*uncore_units_ignore;
1709};
1710
1711static const struct intel_uncore_init_fun nhm_uncore_init __initconst = {
1712	.cpu_init = nhm_uncore_cpu_init,
1713};
1714
1715static const struct intel_uncore_init_fun snb_uncore_init __initconst = {
1716	.cpu_init = snb_uncore_cpu_init,
1717	.pci_init = snb_uncore_pci_init,
1718};
1719
1720static const struct intel_uncore_init_fun ivb_uncore_init __initconst = {
1721	.cpu_init = snb_uncore_cpu_init,
1722	.pci_init = ivb_uncore_pci_init,
1723};
1724
1725static const struct intel_uncore_init_fun hsw_uncore_init __initconst = {
1726	.cpu_init = snb_uncore_cpu_init,
1727	.pci_init = hsw_uncore_pci_init,
1728};
1729
1730static const struct intel_uncore_init_fun bdw_uncore_init __initconst = {
1731	.cpu_init = snb_uncore_cpu_init,
1732	.pci_init = bdw_uncore_pci_init,
1733};
1734
1735static const struct intel_uncore_init_fun snbep_uncore_init __initconst = {
1736	.cpu_init = snbep_uncore_cpu_init,
1737	.pci_init = snbep_uncore_pci_init,
1738};
1739
1740static const struct intel_uncore_init_fun nhmex_uncore_init __initconst = {
1741	.cpu_init = nhmex_uncore_cpu_init,
1742};
1743
1744static const struct intel_uncore_init_fun ivbep_uncore_init __initconst = {
1745	.cpu_init = ivbep_uncore_cpu_init,
1746	.pci_init = ivbep_uncore_pci_init,
1747};
1748
1749static const struct intel_uncore_init_fun hswep_uncore_init __initconst = {
1750	.cpu_init = hswep_uncore_cpu_init,
1751	.pci_init = hswep_uncore_pci_init,
1752};
1753
1754static const struct intel_uncore_init_fun bdx_uncore_init __initconst = {
1755	.cpu_init = bdx_uncore_cpu_init,
1756	.pci_init = bdx_uncore_pci_init,
1757};
1758
1759static const struct intel_uncore_init_fun knl_uncore_init __initconst = {
1760	.cpu_init = knl_uncore_cpu_init,
1761	.pci_init = knl_uncore_pci_init,
1762};
1763
1764static const struct intel_uncore_init_fun skl_uncore_init __initconst = {
1765	.cpu_init = skl_uncore_cpu_init,
1766	.pci_init = skl_uncore_pci_init,
1767};
1768
1769static const struct intel_uncore_init_fun skx_uncore_init __initconst = {
1770	.cpu_init = skx_uncore_cpu_init,
1771	.pci_init = skx_uncore_pci_init,
1772};
1773
1774static const struct intel_uncore_init_fun icl_uncore_init __initconst = {
1775	.cpu_init = icl_uncore_cpu_init,
1776	.pci_init = skl_uncore_pci_init,
1777};
1778
1779static const struct intel_uncore_init_fun tgl_uncore_init __initconst = {
1780	.cpu_init = tgl_uncore_cpu_init,
1781	.mmio_init = tgl_uncore_mmio_init,
1782};
1783
1784static const struct intel_uncore_init_fun tgl_l_uncore_init __initconst = {
1785	.cpu_init = tgl_uncore_cpu_init,
1786	.mmio_init = tgl_l_uncore_mmio_init,
1787};
1788
1789static const struct intel_uncore_init_fun rkl_uncore_init __initconst = {
1790	.cpu_init = tgl_uncore_cpu_init,
1791	.pci_init = skl_uncore_pci_init,
1792};
1793
1794static const struct intel_uncore_init_fun adl_uncore_init __initconst = {
1795	.cpu_init = adl_uncore_cpu_init,
1796	.mmio_init = adl_uncore_mmio_init,
1797};
1798
1799static const struct intel_uncore_init_fun mtl_uncore_init __initconst = {
1800	.cpu_init = mtl_uncore_cpu_init,
1801	.mmio_init = adl_uncore_mmio_init,
1802};
1803
1804static const struct intel_uncore_init_fun lnl_uncore_init __initconst = {
1805	.cpu_init = lnl_uncore_cpu_init,
1806	.mmio_init = lnl_uncore_mmio_init,
1807};
1808
1809static const struct intel_uncore_init_fun ptl_uncore_init __initconst = {
1810	.cpu_init = ptl_uncore_cpu_init,
1811	.mmio_init = ptl_uncore_mmio_init,
1812	.use_discovery = true,
1813};
1814
1815static const struct intel_uncore_init_fun icx_uncore_init __initconst = {
1816	.cpu_init = icx_uncore_cpu_init,
1817	.pci_init = icx_uncore_pci_init,
1818	.mmio_init = icx_uncore_mmio_init,
1819};
1820
1821static const struct intel_uncore_init_fun snr_uncore_init __initconst = {
1822	.cpu_init = snr_uncore_cpu_init,
1823	.pci_init = snr_uncore_pci_init,
1824	.mmio_init = snr_uncore_mmio_init,
1825};
1826
1827static const struct intel_uncore_init_fun spr_uncore_init __initconst = {
1828	.cpu_init = spr_uncore_cpu_init,
1829	.pci_init = spr_uncore_pci_init,
1830	.mmio_init = spr_uncore_mmio_init,
1831	.use_discovery = true,
1832	.uncore_units_ignore = spr_uncore_units_ignore,
1833};
1834
1835static const struct intel_uncore_init_fun gnr_uncore_init __initconst = {
1836	.cpu_init = gnr_uncore_cpu_init,
1837	.pci_init = gnr_uncore_pci_init,
1838	.mmio_init = gnr_uncore_mmio_init,
1839	.use_discovery = true,
1840	.uncore_units_ignore = gnr_uncore_units_ignore,
1841};
1842
1843static const struct intel_uncore_init_fun generic_uncore_init __initconst = {
1844	.cpu_init = intel_uncore_generic_uncore_cpu_init,
1845	.pci_init = intel_uncore_generic_uncore_pci_init,
1846	.mmio_init = intel_uncore_generic_uncore_mmio_init,
1847};
1848
1849static const struct x86_cpu_id intel_uncore_match[] __initconst = {
1850	X86_MATCH_VFM(INTEL_NEHALEM_EP,		&nhm_uncore_init),
1851	X86_MATCH_VFM(INTEL_NEHALEM,		&nhm_uncore_init),
1852	X86_MATCH_VFM(INTEL_WESTMERE,		&nhm_uncore_init),
1853	X86_MATCH_VFM(INTEL_WESTMERE_EP,	&nhm_uncore_init),
1854	X86_MATCH_VFM(INTEL_SANDYBRIDGE,	&snb_uncore_init),
1855	X86_MATCH_VFM(INTEL_IVYBRIDGE,		&ivb_uncore_init),
1856	X86_MATCH_VFM(INTEL_HASWELL,		&hsw_uncore_init),
1857	X86_MATCH_VFM(INTEL_HASWELL_L,		&hsw_uncore_init),
1858	X86_MATCH_VFM(INTEL_HASWELL_G,		&hsw_uncore_init),
1859	X86_MATCH_VFM(INTEL_BROADWELL,		&bdw_uncore_init),
1860	X86_MATCH_VFM(INTEL_BROADWELL_G,	&bdw_uncore_init),
1861	X86_MATCH_VFM(INTEL_SANDYBRIDGE_X,	&snbep_uncore_init),
1862	X86_MATCH_VFM(INTEL_NEHALEM_EX,		&nhmex_uncore_init),
1863	X86_MATCH_VFM(INTEL_WESTMERE_EX,	&nhmex_uncore_init),
1864	X86_MATCH_VFM(INTEL_IVYBRIDGE_X,	&ivbep_uncore_init),
1865	X86_MATCH_VFM(INTEL_HASWELL_X,		&hswep_uncore_init),
1866	X86_MATCH_VFM(INTEL_BROADWELL_X,	&bdx_uncore_init),
1867	X86_MATCH_VFM(INTEL_BROADWELL_D,	&bdx_uncore_init),
1868	X86_MATCH_VFM(INTEL_XEON_PHI_KNL,	&knl_uncore_init),
1869	X86_MATCH_VFM(INTEL_XEON_PHI_KNM,	&knl_uncore_init),
1870	X86_MATCH_VFM(INTEL_SKYLAKE,		&skl_uncore_init),
1871	X86_MATCH_VFM(INTEL_SKYLAKE_L,		&skl_uncore_init),
1872	X86_MATCH_VFM(INTEL_SKYLAKE_X,		&skx_uncore_init),
1873	X86_MATCH_VFM(INTEL_KABYLAKE_L,		&skl_uncore_init),
1874	X86_MATCH_VFM(INTEL_KABYLAKE,		&skl_uncore_init),
1875	X86_MATCH_VFM(INTEL_COMETLAKE_L,	&skl_uncore_init),
1876	X86_MATCH_VFM(INTEL_COMETLAKE,		&skl_uncore_init),
1877	X86_MATCH_VFM(INTEL_ICELAKE_L,		&icl_uncore_init),
1878	X86_MATCH_VFM(INTEL_ICELAKE_NNPI,	&icl_uncore_init),
1879	X86_MATCH_VFM(INTEL_ICELAKE,		&icl_uncore_init),
1880	X86_MATCH_VFM(INTEL_ICELAKE_D,		&icx_uncore_init),
1881	X86_MATCH_VFM(INTEL_ICELAKE_X,		&icx_uncore_init),
1882	X86_MATCH_VFM(INTEL_TIGERLAKE_L,	&tgl_l_uncore_init),
1883	X86_MATCH_VFM(INTEL_TIGERLAKE,		&tgl_uncore_init),
1884	X86_MATCH_VFM(INTEL_ROCKETLAKE,		&rkl_uncore_init),
1885	X86_MATCH_VFM(INTEL_ALDERLAKE,		&adl_uncore_init),
1886	X86_MATCH_VFM(INTEL_ALDERLAKE_L,	&adl_uncore_init),
1887	X86_MATCH_VFM(INTEL_RAPTORLAKE,		&adl_uncore_init),
1888	X86_MATCH_VFM(INTEL_RAPTORLAKE_P,	&adl_uncore_init),
1889	X86_MATCH_VFM(INTEL_RAPTORLAKE_S,	&adl_uncore_init),
1890	X86_MATCH_VFM(INTEL_METEORLAKE,		&mtl_uncore_init),
1891	X86_MATCH_VFM(INTEL_METEORLAKE_L,	&mtl_uncore_init),
1892	X86_MATCH_VFM(INTEL_ARROWLAKE,		&mtl_uncore_init),
1893	X86_MATCH_VFM(INTEL_ARROWLAKE_U,	&mtl_uncore_init),
1894	X86_MATCH_VFM(INTEL_ARROWLAKE_H,	&mtl_uncore_init),
1895	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&lnl_uncore_init),
1896	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,	&ptl_uncore_init),
1897	X86_MATCH_VFM(INTEL_WILDCATLAKE_L,	&ptl_uncore_init),
1898	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X,	&spr_uncore_init),
1899	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,	&spr_uncore_init),
1900	X86_MATCH_VFM(INTEL_GRANITERAPIDS_X,	&gnr_uncore_init),
1901	X86_MATCH_VFM(INTEL_GRANITERAPIDS_D,	&gnr_uncore_init),
1902	X86_MATCH_VFM(INTEL_ATOM_TREMONT_D,	&snr_uncore_init),
1903	X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,	&adl_uncore_init),
1904	X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X,	&gnr_uncore_init),
1905	X86_MATCH_VFM(INTEL_ATOM_CRESTMONT,	&gnr_uncore_init),
1906	X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X,	&gnr_uncore_init),
1907	{},
1908};
1909MODULE_DEVICE_TABLE(x86cpu, intel_uncore_match);
1910
1911static int __init intel_uncore_init(void)
1912{
1913	const struct x86_cpu_id *id;
1914	struct intel_uncore_init_fun *uncore_init;
1915	int pret = 0, cret = 0, mret = 0, ret;
1916
1917	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
1918		return -ENODEV;
1919
1920	__uncore_max_dies =
1921		topology_max_packages() * topology_max_dies_per_package();
1922
1923	id = x86_match_cpu(intel_uncore_match);
1924	if (!id) {
1925		if (!uncore_no_discover && intel_uncore_has_discovery_tables(NULL))
1926			uncore_init = (struct intel_uncore_init_fun *)&generic_uncore_init;
1927		else
1928			return -ENODEV;
1929	} else {
1930		uncore_init = (struct intel_uncore_init_fun *)id->driver_data;
1931		if (uncore_no_discover && uncore_init->use_discovery)
1932			return -ENODEV;
1933		if (uncore_init->use_discovery &&
1934		    !intel_uncore_has_discovery_tables(uncore_init->uncore_units_ignore))
1935			return -ENODEV;
1936	}
1937
1938	if (uncore_init->pci_init) {
1939		pret = uncore_init->pci_init();
1940		if (!pret)
1941			pret = uncore_pci_init();
1942	}
1943
1944	if (uncore_init->cpu_init) {
1945		uncore_init->cpu_init();
1946		cret = uncore_cpu_init();
1947	}
1948
1949	if (uncore_init->mmio_init) {
1950		uncore_init->mmio_init();
1951		mret = uncore_mmio_init();
1952	}
1953
1954	if (cret && pret && mret) {
1955		ret = -ENODEV;
1956		goto free_discovery;
1957	}
1958
1959	/* Install hotplug callbacks to setup the targets for each package */
1960	ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE,
1961				"perf/x86/intel/uncore:online",
1962				uncore_event_cpu_online,
1963				uncore_event_cpu_offline);
1964	if (ret)
1965		goto err;
1966	return 0;
1967
1968err:
1969	uncore_types_exit(uncore_msr_uncores);
1970	uncore_types_exit(uncore_mmio_uncores);
1971	uncore_pci_exit();
1972free_discovery:
1973	intel_uncore_clear_discovery_tables();
1974	return ret;
1975}
1976module_init(intel_uncore_init);
1977
1978static void __exit intel_uncore_exit(void)
1979{
1980	cpuhp_remove_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE);
1981	uncore_types_exit(uncore_msr_uncores);
1982	uncore_types_exit(uncore_mmio_uncores);
1983	uncore_pci_exit();
1984	intel_uncore_clear_discovery_tables();
1985}
1986module_exit(intel_uncore_exit);