book3s_xive.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation.
   4 */
   5
   6#define pr_fmt(fmt) "xive-kvm: " fmt
   7
   8#include <linux/kernel.h>
   9#include <linux/kvm_host.h>
  10#include <linux/err.h>
  11#include <linux/gfp.h>
  12#include <linux/spinlock.h>
  13#include <linux/delay.h>
  14#include <linux/percpu.h>
  15#include <linux/cpumask.h>
  16#include <linux/uaccess.h>
  17#include <linux/irqdomain.h>
  18#include <asm/kvm_book3s.h>
  19#include <asm/kvm_ppc.h>
  20#include <asm/hvcall.h>
  21#include <asm/xics.h>
  22#include <asm/xive.h>
  23#include <asm/xive-regs.h>
  24#include <asm/debug.h>
  25#include <asm/time.h>
  26#include <asm/opal.h>
  27
  28#include <linux/debugfs.h>
  29#include <linux/seq_file.h>
  30
  31#include "book3s_xive.h"
  32
  33#define __x_eoi_page(xd)	((void __iomem *)((xd)->eoi_mmio))
  34#define __x_trig_page(xd)	((void __iomem *)((xd)->trig_mmio))
  35
  36/* Dummy interrupt used when taking interrupts out of a queue in H_CPPR */
  37#define XICS_DUMMY	1
  38
  39static void xive_vm_ack_pending(struct kvmppc_xive_vcpu *xc)
  40{
  41	u8 cppr;
  42	u16 ack;
  43
  44	/*
  45	 * Ensure any previous store to CPPR is ordered vs.
  46	 * the subsequent loads from PIPR or ACK.
  47	 */
  48	eieio();
  49
  50	/* Perform the acknowledge OS to register cycle. */
  51	ack = be16_to_cpu(__raw_readw(xive_tima + TM_SPC_ACK_OS_REG));
  52
  53	/* Synchronize subsequent queue accesses */
  54	mb();
  55
  56	/* XXX Check grouping level */
  57
  58	/* Anything ? */
  59	if (!((ack >> 8) & TM_QW1_NSR_EO))
  60		return;
  61
  62	/* Grab CPPR of the most favored pending interrupt */
  63	cppr = ack & 0xff;
  64	if (cppr < 8)
  65		xc->pending |= 1 << cppr;
  66
  67	/* Check consistency */
  68	if (cppr >= xc->hw_cppr)
  69		pr_warn("KVM-XIVE: CPU %d odd ack CPPR, got %d at %d\n",
  70			smp_processor_id(), cppr, xc->hw_cppr);
  71
  72	/*
  73	 * Update our image of the HW CPPR. We don't yet modify
  74	 * xc->cppr, this will be done as we scan for interrupts
  75	 * in the queues.
  76	 */
  77	xc->hw_cppr = cppr;
  78}
  79
  80static u8 xive_vm_esb_load(struct xive_irq_data *xd, u32 offset)
  81{
  82	u64 val;
  83
  84	if (offset == XIVE_ESB_SET_PQ_10 && xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
  85		offset |= XIVE_ESB_LD_ST_MO;
  86
  87	val = __raw_readq(__x_eoi_page(xd) + offset);
  88#ifdef __LITTLE_ENDIAN__
  89	val >>= 64-8;
  90#endif
  91	return (u8)val;
  92}
  93
  94
  95static void xive_vm_source_eoi(u32 hw_irq, struct xive_irq_data *xd)
  96{
  97	/* If the XIVE supports the new "store EOI facility, use it */
  98	if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
  99		__raw_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI);
 100	else if (xd->flags & XIVE_IRQ_FLAG_LSI) {
 101		/*
 102		 * For LSIs the HW EOI cycle is used rather than PQ bits,
 103		 * as they are automatically re-triggred in HW when still
 104		 * pending.
 105		 */
 106		__raw_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI);
 107	} else {
 108		uint64_t eoi_val;
 109
 110		/*
 111		 * Otherwise for EOI, we use the special MMIO that does
 112		 * a clear of both P and Q and returns the old Q,
 113		 * except for LSIs where we use the "EOI cycle" special
 114		 * load.
 115		 *
 116		 * This allows us to then do a re-trigger if Q was set
 117		 * rather than synthetizing an interrupt in software
 118		 */
 119		eoi_val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_00);
 120
 121		/* Re-trigger if needed */
 122		if ((eoi_val & 1) && __x_trig_page(xd))
 123			__raw_writeq(0, __x_trig_page(xd));
 124	}
 125}
 126
 127enum {
 128	scan_fetch,
 129	scan_poll,
 130	scan_eoi,
 131};
 132
 133static u32 xive_vm_scan_interrupts(struct kvmppc_xive_vcpu *xc,
 134				       u8 pending, int scan_type)
 135{
 136	u32 hirq = 0;
 137	u8 prio = 0xff;
 138
 139	/* Find highest pending priority */
 140	while ((xc->mfrr != 0xff || pending != 0) && hirq == 0) {
 141		struct xive_q *q;
 142		u32 idx, toggle;
 143		__be32 *qpage;
 144
 145		/*
 146		 * If pending is 0 this will return 0xff which is what
 147		 * we want
 148		 */
 149		prio = ffs(pending) - 1;
 150
 151		/* Don't scan past the guest cppr */
 152		if (prio >= xc->cppr || prio > 7) {
 153			if (xc->mfrr < xc->cppr) {
 154				prio = xc->mfrr;
 155				hirq = XICS_IPI;
 156			}
 157			break;
 158		}
 159
 160		/* Grab queue and pointers */
 161		q = &xc->queues[prio];
 162		idx = q->idx;
 163		toggle = q->toggle;
 164
 165		/*
 166		 * Snapshot the queue page. The test further down for EOI
 167		 * must use the same "copy" that was used by __xive_read_eq
 168		 * since qpage can be set concurrently and we don't want
 169		 * to miss an EOI.
 170		 */
 171		qpage = READ_ONCE(q->qpage);
 172
 173skip_ipi:
 174		/*
 175		 * Try to fetch from the queue. Will return 0 for a
 176		 * non-queueing priority (ie, qpage = 0).
 177		 */
 178		hirq = __xive_read_eq(qpage, q->msk, &idx, &toggle);
 179
 180		/*
 181		 * If this was a signal for an MFFR change done by
 182		 * H_IPI we skip it. Additionally, if we were fetching
 183		 * we EOI it now, thus re-enabling reception of a new
 184		 * such signal.
 185		 *
 186		 * We also need to do that if prio is 0 and we had no
 187		 * page for the queue. In this case, we have non-queued
 188		 * IPI that needs to be EOId.
 189		 *
 190		 * This is safe because if we have another pending MFRR
 191		 * change that wasn't observed above, the Q bit will have
 192		 * been set and another occurrence of the IPI will trigger.
 193		 */
 194		if (hirq == XICS_IPI || (prio == 0 && !qpage)) {
 195			if (scan_type == scan_fetch) {
 196				xive_vm_source_eoi(xc->vp_ipi,
 197						       &xc->vp_ipi_data);
 198				q->idx = idx;
 199				q->toggle = toggle;
 200			}
 201			/* Loop back on same queue with updated idx/toggle */
 202			WARN_ON(hirq && hirq != XICS_IPI);
 203			if (hirq)
 204				goto skip_ipi;
 205		}
 206
 207		/* If it's the dummy interrupt, continue searching */
 208		if (hirq == XICS_DUMMY)
 209			goto skip_ipi;
 210
 211		/* Clear the pending bit if the queue is now empty */
 212		if (!hirq) {
 213			pending &= ~(1 << prio);
 214
 215			/*
 216			 * Check if the queue count needs adjusting due to
 217			 * interrupts being moved away.
 218			 */
 219			if (atomic_read(&q->pending_count)) {
 220				int p = atomic_xchg(&q->pending_count, 0);
 221
 222				if (p) {
 223					WARN_ON(p > atomic_read(&q->count));
 224					atomic_sub(p, &q->count);
 225				}
 226			}
 227		}
 228
 229		/*
 230		 * If the most favoured prio we found pending is less
 231		 * favored (or equal) than a pending IPI, we return
 232		 * the IPI instead.
 233		 */
 234		if (prio >= xc->mfrr && xc->mfrr < xc->cppr) {
 235			prio = xc->mfrr;
 236			hirq = XICS_IPI;
 237			break;
 238		}
 239
 240		/* If fetching, update queue pointers */
 241		if (scan_type == scan_fetch) {
 242			q->idx = idx;
 243			q->toggle = toggle;
 244		}
 245	}
 246
 247	/* If we are just taking a "peek", do nothing else */
 248	if (scan_type == scan_poll)
 249		return hirq;
 250
 251	/* Update the pending bits */
 252	xc->pending = pending;
 253
 254	/*
 255	 * If this is an EOI that's it, no CPPR adjustment done here,
 256	 * all we needed was cleanup the stale pending bits and check
 257	 * if there's anything left.
 258	 */
 259	if (scan_type == scan_eoi)
 260		return hirq;
 261
 262	/*
 263	 * If we found an interrupt, adjust what the guest CPPR should
 264	 * be as if we had just fetched that interrupt from HW.
 265	 *
 266	 * Note: This can only make xc->cppr smaller as the previous
 267	 * loop will only exit with hirq != 0 if prio is lower than
 268	 * the current xc->cppr. Thus we don't need to re-check xc->mfrr
 269	 * for pending IPIs.
 270	 */
 271	if (hirq)
 272		xc->cppr = prio;
 273	/*
 274	 * If it was an IPI the HW CPPR might have been lowered too much
 275	 * as the HW interrupt we use for IPIs is routed to priority 0.
 276	 *
 277	 * We re-sync it here.
 278	 */
 279	if (xc->cppr != xc->hw_cppr) {
 280		xc->hw_cppr = xc->cppr;
 281		__raw_writeb(xc->cppr, xive_tima + TM_QW1_OS + TM_CPPR);
 282	}
 283
 284	return hirq;
 285}
 286
 287static unsigned long xive_vm_h_xirr(struct kvm_vcpu *vcpu)
 288{
 289	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
 290	u8 old_cppr;
 291	u32 hirq;
 292
 293	pr_devel("H_XIRR\n");
 294
 295	xc->stat_vm_h_xirr++;
 296
 297	/* First collect pending bits from HW */
 298	xive_vm_ack_pending(xc);
 299
 300	pr_devel(" new pending=0x%02x hw_cppr=%d cppr=%d\n",
 301		 xc->pending, xc->hw_cppr, xc->cppr);
 302
 303	/* Grab previous CPPR and reverse map it */
 304	old_cppr = xive_prio_to_guest(xc->cppr);
 305
 306	/* Scan for actual interrupts */
 307	hirq = xive_vm_scan_interrupts(xc, xc->pending, scan_fetch);
 308
 309	pr_devel(" got hirq=0x%x hw_cppr=%d cppr=%d\n",
 310		 hirq, xc->hw_cppr, xc->cppr);
 311
 312	/* That should never hit */
 313	if (hirq & 0xff000000)
 314		pr_warn("XIVE: Weird guest interrupt number 0x%08x\n", hirq);
 315
 316	/*
 317	 * XXX We could check if the interrupt is masked here and
 318	 * filter it. If we chose to do so, we would need to do:
 319	 *
 320	 *    if (masked) {
 321	 *        lock();
 322	 *        if (masked) {
 323	 *            old_Q = true;
 324	 *            hirq = 0;
 325	 *        }
 326	 *        unlock();
 327	 *    }
 328	 */
 329
 330	/* Return interrupt and old CPPR in GPR4 */
 331	kvmppc_set_gpr(vcpu, 4, hirq | (old_cppr << 24));
 332
 333	return H_SUCCESS;
 334}
 335
 336static unsigned long xive_vm_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server)
 337{
 338	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
 339	u8 pending = xc->pending;
 340	u32 hirq;
 341
 342	pr_devel("H_IPOLL(server=%ld)\n", server);
 343
 344	xc->stat_vm_h_ipoll++;
 345
 346	/* Grab the target VCPU if not the current one */
 347	if (xc->server_num != server) {
 348		vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
 349		if (!vcpu)
 350			return H_PARAMETER;
 351		xc = vcpu->arch.xive_vcpu;
 352
 353		/* Scan all priorities */
 354		pending = 0xff;
 355	} else {
 356		/* Grab pending interrupt if any */
 357		__be64 qw1 = __raw_readq(xive_tima + TM_QW1_OS);
 358		u8 pipr = be64_to_cpu(qw1) & 0xff;
 359
 360		if (pipr < 8)
 361			pending |= 1 << pipr;
 362	}
 363
 364	hirq = xive_vm_scan_interrupts(xc, pending, scan_poll);
 365
 366	/* Return interrupt and old CPPR in GPR4 */
 367	kvmppc_set_gpr(vcpu, 4, hirq | (xc->cppr << 24));
 368
 369	return H_SUCCESS;
 370}
 371
 372static void xive_vm_push_pending_to_hw(struct kvmppc_xive_vcpu *xc)
 373{
 374	u8 pending, prio;
 375
 376	pending = xc->pending;
 377	if (xc->mfrr != 0xff) {
 378		if (xc->mfrr < 8)
 379			pending |= 1 << xc->mfrr;
 380		else
 381			pending |= 0x80;
 382	}
 383	if (!pending)
 384		return;
 385	prio = ffs(pending) - 1;
 386
 387	__raw_writeb(prio, xive_tima + TM_SPC_SET_OS_PENDING);
 388}
 389
 390static void xive_vm_scan_for_rerouted_irqs(struct kvmppc_xive *xive,
 391					       struct kvmppc_xive_vcpu *xc)
 392{
 393	unsigned int prio;
 394
 395	/* For each priority that is now masked */
 396	for (prio = xc->cppr; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
 397		struct xive_q *q = &xc->queues[prio];
 398		struct kvmppc_xive_irq_state *state;
 399		struct kvmppc_xive_src_block *sb;
 400		u32 idx, toggle, entry, irq, hw_num;
 401		struct xive_irq_data *xd;
 402		__be32 *qpage;
 403		u16 src;
 404
 405		idx = q->idx;
 406		toggle = q->toggle;
 407		qpage = READ_ONCE(q->qpage);
 408		if (!qpage)
 409			continue;
 410
 411		/* For each interrupt in the queue */
 412		for (;;) {
 413			entry = be32_to_cpup(qpage + idx);
 414
 415			/* No more ? */
 416			if ((entry >> 31) == toggle)
 417				break;
 418			irq = entry & 0x7fffffff;
 419
 420			/* Skip dummies and IPIs */
 421			if (irq == XICS_DUMMY || irq == XICS_IPI)
 422				goto next;
 423			sb = kvmppc_xive_find_source(xive, irq, &src);
 424			if (!sb)
 425				goto next;
 426			state = &sb->irq_state[src];
 427
 428			/* Has it been rerouted ? */
 429			if (xc->server_num == state->act_server)
 430				goto next;
 431
 432			/*
 433			 * Allright, it *has* been re-routed, kill it from
 434			 * the queue.
 435			 */
 436			qpage[idx] = cpu_to_be32((entry & 0x80000000) | XICS_DUMMY);
 437
 438			/* Find the HW interrupt */
 439			kvmppc_xive_select_irq(state, &hw_num, &xd);
 440
 441			/* If it's not an LSI, set PQ to 11 the EOI will force a resend */
 442			if (!(xd->flags & XIVE_IRQ_FLAG_LSI))
 443				xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
 444
 445			/* EOI the source */
 446			xive_vm_source_eoi(hw_num, xd);
 447
 448next:
 449			idx = (idx + 1) & q->msk;
 450			if (idx == 0)
 451				toggle ^= 1;
 452		}
 453	}
 454}
 455
 456static int xive_vm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr)
 457{
 458	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
 459	struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
 460	u8 old_cppr;
 461
 462	pr_devel("H_CPPR(cppr=%ld)\n", cppr);
 463
 464	xc->stat_vm_h_cppr++;
 465
 466	/* Map CPPR */
 467	cppr = xive_prio_from_guest(cppr);
 468
 469	/* Remember old and update SW state */
 470	old_cppr = xc->cppr;
 471	xc->cppr = cppr;
 472
 473	/*
 474	 * Order the above update of xc->cppr with the subsequent
 475	 * read of xc->mfrr inside push_pending_to_hw()
 476	 */
 477	smp_mb();
 478
 479	if (cppr > old_cppr) {
 480		/*
 481		 * We are masking less, we need to look for pending things
 482		 * to deliver and set VP pending bits accordingly to trigger
 483		 * a new interrupt otherwise we might miss MFRR changes for
 484		 * which we have optimized out sending an IPI signal.
 485		 */
 486		xive_vm_push_pending_to_hw(xc);
 487	} else {
 488		/*
 489		 * We are masking more, we need to check the queue for any
 490		 * interrupt that has been routed to another CPU, take
 491		 * it out (replace it with the dummy) and retrigger it.
 492		 *
 493		 * This is necessary since those interrupts may otherwise
 494		 * never be processed, at least not until this CPU restores
 495		 * its CPPR.
 496		 *
 497		 * This is in theory racy vs. HW adding new interrupts to
 498		 * the queue. In practice this works because the interesting
 499		 * cases are when the guest has done a set_xive() to move the
 500		 * interrupt away, which flushes the xive, followed by the
 501		 * target CPU doing a H_CPPR. So any new interrupt coming into
 502		 * the queue must still be routed to us and isn't a source
 503		 * of concern.
 504		 */
 505		xive_vm_scan_for_rerouted_irqs(xive, xc);
 506	}
 507
 508	/* Apply new CPPR */
 509	xc->hw_cppr = cppr;
 510	__raw_writeb(cppr, xive_tima + TM_QW1_OS + TM_CPPR);
 511
 512	return H_SUCCESS;
 513}
 514
 515static int xive_vm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
 516{
 517	struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
 518	struct kvmppc_xive_src_block *sb;
 519	struct kvmppc_xive_irq_state *state;
 520	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
 521	struct xive_irq_data *xd;
 522	u8 new_cppr = xirr >> 24;
 523	u32 irq = xirr & 0x00ffffff, hw_num;
 524	u16 src;
 525	int rc = 0;
 526
 527	pr_devel("H_EOI(xirr=%08lx)\n", xirr);
 528
 529	xc->stat_vm_h_eoi++;
 530
 531	xc->cppr = xive_prio_from_guest(new_cppr);
 532
 533	/*
 534	 * IPIs are synthesized from MFRR and thus don't need
 535	 * any special EOI handling. The underlying interrupt
 536	 * used to signal MFRR changes is EOId when fetched from
 537	 * the queue.
 538	 */
 539	if (irq == XICS_IPI || irq == 0) {
 540		/*
 541		 * This barrier orders the setting of xc->cppr vs.
 542		 * subsequent test of xc->mfrr done inside
 543		 * scan_interrupts and push_pending_to_hw
 544		 */
 545		smp_mb();
 546		goto bail;
 547	}
 548
 549	/* Find interrupt source */
 550	sb = kvmppc_xive_find_source(xive, irq, &src);
 551	if (!sb) {
 552		pr_devel(" source not found !\n");
 553		rc = H_PARAMETER;
 554		/* Same as above */
 555		smp_mb();
 556		goto bail;
 557	}
 558	state = &sb->irq_state[src];
 559	kvmppc_xive_select_irq(state, &hw_num, &xd);
 560
 561	state->in_eoi = true;
 562
 563	/*
 564	 * This barrier orders both setting of in_eoi above vs,
 565	 * subsequent test of guest_priority, and the setting
 566	 * of xc->cppr vs. subsequent test of xc->mfrr done inside
 567	 * scan_interrupts and push_pending_to_hw
 568	 */
 569	smp_mb();
 570
 571again:
 572	if (state->guest_priority == MASKED) {
 573		arch_spin_lock(&sb->lock);
 574		if (state->guest_priority != MASKED) {
 575			arch_spin_unlock(&sb->lock);
 576			goto again;
 577		}
 578		pr_devel(" EOI on saved P...\n");
 579
 580		/* Clear old_p, that will cause unmask to perform an EOI */
 581		state->old_p = false;
 582
 583		arch_spin_unlock(&sb->lock);
 584	} else {
 585		pr_devel(" EOI on source...\n");
 586
 587		/* Perform EOI on the source */
 588		xive_vm_source_eoi(hw_num, xd);
 589
 590		/* If it's an emulated LSI, check level and resend */
 591		if (state->lsi && state->asserted)
 592			__raw_writeq(0, __x_trig_page(xd));
 593
 594	}
 595
 596	/*
 597	 * This barrier orders the above guest_priority check
 598	 * and spin_lock/unlock with clearing in_eoi below.
 599	 *
 600	 * It also has to be a full mb() as it must ensure
 601	 * the MMIOs done in source_eoi() are completed before
 602	 * state->in_eoi is visible.
 603	 */
 604	mb();
 605	state->in_eoi = false;
 606bail:
 607
 608	/* Re-evaluate pending IRQs and update HW */
 609	xive_vm_scan_interrupts(xc, xc->pending, scan_eoi);
 610	xive_vm_push_pending_to_hw(xc);
 611	pr_devel(" after scan pending=%02x\n", xc->pending);
 612
 613	/* Apply new CPPR */
 614	xc->hw_cppr = xc->cppr;
 615	__raw_writeb(xc->cppr, xive_tima + TM_QW1_OS + TM_CPPR);
 616
 617	return rc;
 618}
 619
 620static int xive_vm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
 621			       unsigned long mfrr)
 622{
 623	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
 624
 625	pr_devel("H_IPI(server=%08lx,mfrr=%ld)\n", server, mfrr);
 626
 627	xc->stat_vm_h_ipi++;
 628
 629	/* Find target */
 630	vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
 631	if (!vcpu)
 632		return H_PARAMETER;
 633	xc = vcpu->arch.xive_vcpu;
 634
 635	/* Locklessly write over MFRR */
 636	xc->mfrr = mfrr;
 637
 638	/*
 639	 * The load of xc->cppr below and the subsequent MMIO store
 640	 * to the IPI must happen after the above mfrr update is
 641	 * globally visible so that:
 642	 *
 643	 * - Synchronize with another CPU doing an H_EOI or a H_CPPR
 644	 *   updating xc->cppr then reading xc->mfrr.
 645	 *
 646	 * - The target of the IPI sees the xc->mfrr update
 647	 */
 648	mb();
 649
 650	/* Shoot the IPI if most favored than target cppr */
 651	if (mfrr < xc->cppr)
 652		__raw_writeq(0, __x_trig_page(&xc->vp_ipi_data));
 653
 654	return H_SUCCESS;
 655}
 656
 657/*
 658 * We leave a gap of a couple of interrupts in the queue to
 659 * account for the IPI and additional safety guard.
 660 */
 661#define XIVE_Q_GAP	2
 662
 663static bool kvmppc_xive_vcpu_has_save_restore(struct kvm_vcpu *vcpu)
 664{
 665	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
 666
 667	/* Check enablement at VP level */
 668	return xc->vp_cam & TM_QW1W2_HO;
 669}
 670
 671bool kvmppc_xive_check_save_restore(struct kvm_vcpu *vcpu)
 672{
 673	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
 674	struct kvmppc_xive *xive = xc->xive;
 675
 676	if (xive->flags & KVMPPC_XIVE_FLAG_SAVE_RESTORE)
 677		return kvmppc_xive_vcpu_has_save_restore(vcpu);
 678
 679	return true;
 680}
 681
 682/*
 683 * Push a vcpu's context to the XIVE on guest entry.
 684 * This assumes we are in virtual mode (MMU on)
 685 */
 686void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu)
 687{
 688	void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
 689	u64 pq;
 690
 691	/*
 692	 * Nothing to do if the platform doesn't have a XIVE
 693	 * or this vCPU doesn't have its own XIVE context
 694	 * (e.g. because it's not using an in-kernel interrupt controller).
 695	 */
 696	if (!tima || !vcpu->arch.xive_cam_word)
 697		return;
 698
 699	eieio();
 700	if (!kvmppc_xive_vcpu_has_save_restore(vcpu))
 701		__raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS);
 702	__raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2);
 703	vcpu->arch.xive_pushed = 1;
 704	eieio();
 705
 706	/*
 707	 * We clear the irq_pending flag. There is a small chance of a
 708	 * race vs. the escalation interrupt happening on another
 709	 * processor setting it again, but the only consequence is to
 710	 * cause a spurious wakeup on the next H_CEDE, which is not an
 711	 * issue.
 712	 */
 713	vcpu->arch.irq_pending = 0;
 714
 715	/*
 716	 * In single escalation mode, if the escalation interrupt is
 717	 * on, we mask it.
 718	 */
 719	if (vcpu->arch.xive_esc_on) {
 720		pq = __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
 721						  XIVE_ESB_SET_PQ_01));
 722		mb();
 723
 724		/*
 725		 * We have a possible subtle race here: The escalation
 726		 * interrupt might have fired and be on its way to the
 727		 * host queue while we mask it, and if we unmask it
 728		 * early enough (re-cede right away), there is a
 729		 * theoretical possibility that it fires again, thus
 730		 * landing in the target queue more than once which is
 731		 * a big no-no.
 732		 *
 733		 * Fortunately, solving this is rather easy. If the
 734		 * above load setting PQ to 01 returns a previous
 735		 * value where P is set, then we know the escalation
 736		 * interrupt is somewhere on its way to the host. In
 737		 * that case we simply don't clear the xive_esc_on
 738		 * flag below. It will be eventually cleared by the
 739		 * handler for the escalation interrupt.
 740		 *
 741		 * Then, when doing a cede, we check that flag again
 742		 * before re-enabling the escalation interrupt, and if
 743		 * set, we abort the cede.
 744		 */
 745		if (!(pq & XIVE_ESB_VAL_P))
 746			/* Now P is 0, we can clear the flag */
 747			vcpu->arch.xive_esc_on = 0;
 748	}
 749}
 750EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu);
 751
 752/*
 753 * Pull a vcpu's context from the XIVE on guest exit.
 754 * This assumes we are in virtual mode (MMU on)
 755 */
 756void kvmppc_xive_pull_vcpu(struct kvm_vcpu *vcpu)
 757{
 758	void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
 759
 760	if (!vcpu->arch.xive_pushed)
 761		return;
 762
 763	/*
 764	 * Should not have been pushed if there is no tima
 765	 */
 766	if (WARN_ON(!tima))
 767		return;
 768
 769	eieio();
 770	/* First load to pull the context, we ignore the value */
 771	__raw_readl(tima + TM_SPC_PULL_OS_CTX);
 772	/* Second load to recover the context state (Words 0 and 1) */
 773	if (!kvmppc_xive_vcpu_has_save_restore(vcpu))
 774		vcpu->arch.xive_saved_state.w01 = __raw_readq(tima + TM_QW1_OS);
 775
 776	/* Fixup some of the state for the next load */
 777	vcpu->arch.xive_saved_state.lsmfb = 0;
 778	vcpu->arch.xive_saved_state.ack = 0xff;
 779	vcpu->arch.xive_pushed = 0;
 780	eieio();
 781}
 782EXPORT_SYMBOL_GPL(kvmppc_xive_pull_vcpu);
 783
 784bool kvmppc_xive_rearm_escalation(struct kvm_vcpu *vcpu)
 785{
 786	void __iomem *esc_vaddr = (void __iomem *)vcpu->arch.xive_esc_vaddr;
 787	bool ret = true;
 788
 789	if (!esc_vaddr)
 790		return ret;
 791
 792	/* we are using XIVE with single escalation */
 793
 794	if (vcpu->arch.xive_esc_on) {
 795		/*
 796		 * If we still have a pending escalation, abort the cede,
 797		 * and we must set PQ to 10 rather than 00 so that we don't
 798		 * potentially end up with two entries for the escalation
 799		 * interrupt in the XIVE interrupt queue.  In that case
 800		 * we also don't want to set xive_esc_on to 1 here in
 801		 * case we race with xive_esc_irq().
 802		 */
 803		ret = false;
 804		/*
 805		 * The escalation interrupts are special as we don't EOI them.
 806		 * There is no need to use the load-after-store ordering offset
 807		 * to set PQ to 10 as we won't use StoreEOI.
 808		 */
 809		__raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_10);
 810	} else {
 811		vcpu->arch.xive_esc_on = true;
 812		mb();
 813		__raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_00);
 814	}
 815	mb();
 816
 817	return ret;
 818}
 819EXPORT_SYMBOL_GPL(kvmppc_xive_rearm_escalation);
 820
 821/*
 822 * This is a simple trigger for a generic XIVE IRQ. This must
 823 * only be called for interrupts that support a trigger page
 824 */
 825static bool xive_irq_trigger(struct xive_irq_data *xd)
 826{
 827	/* This should be only for MSIs */
 828	if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))
 829		return false;
 830
 831	/* Those interrupts should always have a trigger page */
 832	if (WARN_ON(!xd->trig_mmio))
 833		return false;
 834
 835	out_be64(xd->trig_mmio, 0);
 836
 837	return true;
 838}
 839
 840static irqreturn_t xive_esc_irq(int irq, void *data)
 841{
 842	struct kvm_vcpu *vcpu = data;
 843
 844	vcpu->arch.irq_pending = 1;
 845	smp_mb();
 846	if (vcpu->arch.ceded || vcpu->arch.nested)
 847		kvmppc_fast_vcpu_kick(vcpu);
 848
 849	/* Since we have the no-EOI flag, the interrupt is effectively
 850	 * disabled now. Clearing xive_esc_on means we won't bother
 851	 * doing so on the next entry.
 852	 *
 853	 * This also allows the entry code to know that if a PQ combination
 854	 * of 10 is observed while xive_esc_on is true, it means the queue
 855	 * contains an unprocessed escalation interrupt. We don't make use of
 856	 * that knowledge today but might (see comment in book3s_hv_rmhandler.S)
 857	 */
 858	vcpu->arch.xive_esc_on = false;
 859
 860	/* This orders xive_esc_on = false vs. subsequent stale_p = true */
 861	smp_wmb();	/* goes with smp_mb() in cleanup_single_escalation */
 862
 863	return IRQ_HANDLED;
 864}
 865
 866int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,
 867				  bool single_escalation)
 868{
 869	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
 870	struct xive_q *q = &xc->queues[prio];
 871	char *name = NULL;
 872	int rc;
 873
 874	/* Already there ? */
 875	if (xc->esc_virq[prio])
 876		return 0;
 877
 878	/* Hook up the escalation interrupt */
 879	xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq);
 880	if (!xc->esc_virq[prio]) {
 881		pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n",
 882		       prio, xc->server_num);
 883		return -EIO;
 884	}
 885
 886	if (single_escalation)
 887		name = kasprintf(GFP_KERNEL, "kvm-%lld-%d",
 888				 vcpu->kvm->arch.lpid, xc->server_num);
 889	else
 890		name = kasprintf(GFP_KERNEL, "kvm-%lld-%d-%d",
 891				 vcpu->kvm->arch.lpid, xc->server_num, prio);
 892	if (!name) {
 893		pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n",
 894		       prio, xc->server_num);
 895		rc = -ENOMEM;
 896		goto error;
 897	}
 898
 899	pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio);
 900
 901	rc = request_irq(xc->esc_virq[prio], xive_esc_irq,
 902			 IRQF_NO_THREAD, name, vcpu);
 903	if (rc) {
 904		pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n",
 905		       prio, xc->server_num);
 906		goto error;
 907	}
 908	xc->esc_virq_names[prio] = name;
 909
 910	/* In single escalation mode, we grab the ESB MMIO of the
 911	 * interrupt and mask it. Also populate the VCPU v/raddr
 912	 * of the ESB page for use by asm entry/exit code. Finally
 913	 * set the XIVE_IRQ_FLAG_NO_EOI flag which will prevent the
 914	 * core code from performing an EOI on the escalation
 915	 * interrupt, thus leaving it effectively masked after
 916	 * it fires once.
 917	 */
 918	if (single_escalation) {
 919		struct xive_irq_data *xd = irq_get_chip_data(xc->esc_virq[prio]);
 920
 921		xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
 922		vcpu->arch.xive_esc_raddr = xd->eoi_page;
 923		vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio;
 924		xd->flags |= XIVE_IRQ_FLAG_NO_EOI;
 925	}
 926
 927	return 0;
 928error:
 929	irq_dispose_mapping(xc->esc_virq[prio]);
 930	xc->esc_virq[prio] = 0;
 931	kfree(name);
 932	return rc;
 933}
 934
 935static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio)
 936{
 937	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
 938	struct kvmppc_xive *xive = xc->xive;
 939	struct xive_q *q =  &xc->queues[prio];
 940	void *qpage;
 941	int rc;
 942
 943	if (WARN_ON(q->qpage))
 944		return 0;
 945
 946	/* Allocate the queue and retrieve infos on current node for now */
 947	qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order);
 948	if (!qpage) {
 949		pr_err("Failed to allocate queue %d for VCPU %d\n",
 950		       prio, xc->server_num);
 951		return -ENOMEM;
 952	}
 953	memset(qpage, 0, 1 << xive->q_order);
 954
 955	/*
 956	 * Reconfigure the queue. This will set q->qpage only once the
 957	 * queue is fully configured. This is a requirement for prio 0
 958	 * as we will stop doing EOIs for every IPI as soon as we observe
 959	 * qpage being non-NULL, and instead will only EOI when we receive
 960	 * corresponding queue 0 entries
 961	 */
 962	rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage,
 963					 xive->q_order, true);
 964	if (rc)
 965		pr_err("Failed to configure queue %d for VCPU %d\n",
 966		       prio, xc->server_num);
 967	return rc;
 968}
 969
 970/* Called with xive->lock held */
 971static int xive_check_provisioning(struct kvm *kvm, u8 prio)
 972{
 973	struct kvmppc_xive *xive = kvm->arch.xive;
 974	struct kvm_vcpu *vcpu;
 975	unsigned long i;
 976	int rc;
 977
 978	lockdep_assert_held(&xive->lock);
 979
 980	/* Already provisioned ? */
 981	if (xive->qmap & (1 << prio))
 982		return 0;
 983
 984	pr_devel("Provisioning prio... %d\n", prio);
 985
 986	/* Provision each VCPU and enable escalations if needed */
 987	kvm_for_each_vcpu(i, vcpu, kvm) {
 988		if (!vcpu->arch.xive_vcpu)
 989			continue;
 990		rc = xive_provision_queue(vcpu, prio);
 991		if (rc == 0 && !kvmppc_xive_has_single_escalation(xive))
 992			kvmppc_xive_attach_escalation(vcpu, prio,
 993						      kvmppc_xive_has_single_escalation(xive));
 994		if (rc)
 995			return rc;
 996	}
 997
 998	/* Order previous stores and mark it as provisioned */
 999	mb();
1000	xive->qmap |= (1 << prio);
1001	return 0;
1002}
1003
1004static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio)
1005{
1006	struct kvm_vcpu *vcpu;
1007	struct kvmppc_xive_vcpu *xc;
1008	struct xive_q *q;
1009
1010	/* Locate target server */
1011	vcpu = kvmppc_xive_find_server(kvm, server);
1012	if (!vcpu) {
1013		pr_warn("%s: Can't find server %d\n", __func__, server);
1014		return;
1015	}
1016	xc = vcpu->arch.xive_vcpu;
1017	if (WARN_ON(!xc))
1018		return;
1019
1020	q = &xc->queues[prio];
1021	atomic_inc(&q->pending_count);
1022}
1023
1024static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio)
1025{
1026	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1027	struct xive_q *q;
1028	u32 max;
1029
1030	if (WARN_ON(!xc))
1031		return -ENXIO;
1032	if (!xc->valid)
1033		return -ENXIO;
1034
1035	q = &xc->queues[prio];
1036	if (WARN_ON(!q->qpage))
1037		return -ENXIO;
1038
1039	/* Calculate max number of interrupts in that queue. */
1040	max = (q->msk + 1) - XIVE_Q_GAP;
1041	return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY;
1042}
1043
1044int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio)
1045{
1046	struct kvm_vcpu *vcpu;
1047	unsigned long i;
1048	int rc;
1049
1050	/* Locate target server */
1051	vcpu = kvmppc_xive_find_server(kvm, *server);
1052	if (!vcpu) {
1053		pr_devel("Can't find server %d\n", *server);
1054		return -EINVAL;
1055	}
1056
1057	pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio);
1058
1059	/* Try pick it */
1060	rc = xive_try_pick_queue(vcpu, prio);
1061	if (rc == 0)
1062		return rc;
1063
1064	pr_devel(" .. failed, looking up candidate...\n");
1065
1066	/* Failed, pick another VCPU */
1067	kvm_for_each_vcpu(i, vcpu, kvm) {
1068		if (!vcpu->arch.xive_vcpu)
1069			continue;
1070		rc = xive_try_pick_queue(vcpu, prio);
1071		if (rc == 0) {
1072			*server = vcpu->arch.xive_vcpu->server_num;
1073			pr_devel("  found on 0x%x/%d\n", *server, prio);
1074			return rc;
1075		}
1076	}
1077	pr_devel("  no available target !\n");
1078
1079	/* No available target ! */
1080	return -EBUSY;
1081}
1082
1083static u8 xive_lock_and_mask(struct kvmppc_xive *xive,
1084			     struct kvmppc_xive_src_block *sb,
1085			     struct kvmppc_xive_irq_state *state)
1086{
1087	struct xive_irq_data *xd;
1088	u32 hw_num;
1089	u8 old_prio;
1090	u64 val;
1091
1092	/*
1093	 * Take the lock, set masked, try again if racing
1094	 * with H_EOI
1095	 */
1096	for (;;) {
1097		arch_spin_lock(&sb->lock);
1098		old_prio = state->guest_priority;
1099		state->guest_priority = MASKED;
1100		mb();
1101		if (!state->in_eoi)
1102			break;
1103		state->guest_priority = old_prio;
1104		arch_spin_unlock(&sb->lock);
1105	}
1106
1107	/* No change ? Bail */
1108	if (old_prio == MASKED)
1109		return old_prio;
1110
1111	/* Get the right irq */
1112	kvmppc_xive_select_irq(state, &hw_num, &xd);
1113
1114	/* Set PQ to 10, return old P and old Q and remember them */
1115	val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10);
1116	state->old_p = !!(val & 2);
1117	state->old_q = !!(val & 1);
1118
1119	/*
1120	 * Synchronize hardware to sensure the queues are updated when
1121	 * masking
1122	 */
1123	xive_native_sync_source(hw_num);
1124
1125	return old_prio;
1126}
1127
1128static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb,
1129				 struct kvmppc_xive_irq_state *state)
1130{
1131	/*
1132	 * Take the lock try again if racing with H_EOI
1133	 */
1134	for (;;) {
1135		arch_spin_lock(&sb->lock);
1136		if (!state->in_eoi)
1137			break;
1138		arch_spin_unlock(&sb->lock);
1139	}
1140}
1141
1142static void xive_finish_unmask(struct kvmppc_xive *xive,
1143			       struct kvmppc_xive_src_block *sb,
1144			       struct kvmppc_xive_irq_state *state,
1145			       u8 prio)
1146{
1147	struct xive_irq_data *xd;
1148	u32 hw_num;
1149
1150	/* If we aren't changing a thing, move on */
1151	if (state->guest_priority != MASKED)
1152		goto bail;
1153
1154	/* Get the right irq */
1155	kvmppc_xive_select_irq(state, &hw_num, &xd);
1156
1157	/* Old Q set, set PQ to 11 */
1158	if (state->old_q)
1159		xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
1160
1161	/*
1162	 * If not old P, then perform an "effective" EOI,
1163	 * on the source. This will handle the cases where
1164	 * FW EOI is needed.
1165	 */
1166	if (!state->old_p)
1167		xive_vm_source_eoi(hw_num, xd);
1168
1169	/* Synchronize ordering and mark unmasked */
1170	mb();
1171bail:
1172	state->guest_priority = prio;
1173}
1174
1175/*
1176 * Target an interrupt to a given server/prio, this will fallback
1177 * to another server if necessary and perform the HW targetting
1178 * updates as needed
1179 *
1180 * NOTE: Must be called with the state lock held
1181 */
1182static int xive_target_interrupt(struct kvm *kvm,
1183				 struct kvmppc_xive_irq_state *state,
1184				 u32 server, u8 prio)
1185{
1186	struct kvmppc_xive *xive = kvm->arch.xive;
1187	u32 hw_num;
1188	int rc;
1189
1190	/*
1191	 * This will return a tentative server and actual
1192	 * priority. The count for that new target will have
1193	 * already been incremented.
1194	 */
1195	rc = kvmppc_xive_select_target(kvm, &server, prio);
1196
1197	/*
1198	 * We failed to find a target ? Not much we can do
1199	 * at least until we support the GIQ.
1200	 */
1201	if (rc)
1202		return rc;
1203
1204	/*
1205	 * Increment the old queue pending count if there
1206	 * was one so that the old queue count gets adjusted later
1207	 * when observed to be empty.
1208	 */
1209	if (state->act_priority != MASKED)
1210		xive_inc_q_pending(kvm,
1211				   state->act_server,
1212				   state->act_priority);
1213	/*
1214	 * Update state and HW
1215	 */
1216	state->act_priority = prio;
1217	state->act_server = server;
1218
1219	/* Get the right irq */
1220	kvmppc_xive_select_irq(state, &hw_num, NULL);
1221
1222	return xive_native_configure_irq(hw_num,
1223					 kvmppc_xive_vp(xive, server),
1224					 prio, state->number);
1225}
1226
1227/*
1228 * Targetting rules: In order to avoid losing track of
1229 * pending interrupts across mask and unmask, which would
1230 * allow queue overflows, we implement the following rules:
1231 *
1232 *  - Unless it was never enabled (or we run out of capacity)
1233 *    an interrupt is always targetted at a valid server/queue
1234 *    pair even when "masked" by the guest. This pair tends to
1235 *    be the last one used but it can be changed under some
1236 *    circumstances. That allows us to separate targetting
1237 *    from masking, we only handle accounting during (re)targetting,
1238 *    this also allows us to let an interrupt drain into its target
1239 *    queue after masking, avoiding complex schemes to remove
1240 *    interrupts out of remote processor queues.
1241 *
1242 *  - When masking, we set PQ to 10 and save the previous value
1243 *    of P and Q.
1244 *
1245 *  - When unmasking, if saved Q was set, we set PQ to 11
1246 *    otherwise we leave PQ to the HW state which will be either
1247 *    10 if nothing happened or 11 if the interrupt fired while
1248 *    masked. Effectively we are OR'ing the previous Q into the
1249 *    HW Q.
1250 *
1251 *    Then if saved P is clear, we do an effective EOI (Q->P->Trigger)
1252 *    which will unmask the interrupt and shoot a new one if Q was
1253 *    set.
1254 *
1255 *    Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11,
1256 *    effectively meaning an H_EOI from the guest is still expected
1257 *    for that interrupt).
1258 *
1259 *  - If H_EOI occurs while masked, we clear the saved P.
1260 *
1261 *  - When changing target, we account on the new target and
1262 *    increment a separate "pending" counter on the old one.
1263 *    This pending counter will be used to decrement the old
1264 *    target's count when its queue has been observed empty.
1265 */
1266
1267int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
1268			 u32 priority)
1269{
1270	struct kvmppc_xive *xive = kvm->arch.xive;
1271	struct kvmppc_xive_src_block *sb;
1272	struct kvmppc_xive_irq_state *state;
1273	u8 new_act_prio;
1274	int rc = 0;
1275	u16 idx;
1276
1277	if (!xive)
1278		return -ENODEV;
1279
1280	pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n",
1281		 irq, server, priority);
1282
1283	/* First, check provisioning of queues */
1284	if (priority != MASKED) {
1285		mutex_lock(&xive->lock);
1286		rc = xive_check_provisioning(xive->kvm,
1287			      xive_prio_from_guest(priority));
1288		mutex_unlock(&xive->lock);
1289	}
1290	if (rc) {
1291		pr_devel("  provisioning failure %d !\n", rc);
1292		return rc;
1293	}
1294
1295	sb = kvmppc_xive_find_source(xive, irq, &idx);
1296	if (!sb)
1297		return -EINVAL;
1298	state = &sb->irq_state[idx];
1299
1300	/*
1301	 * We first handle masking/unmasking since the locking
1302	 * might need to be retried due to EOIs, we'll handle
1303	 * targetting changes later. These functions will return
1304	 * with the SB lock held.
1305	 *
1306	 * xive_lock_and_mask() will also set state->guest_priority
1307	 * but won't otherwise change other fields of the state.
1308	 *
1309	 * xive_lock_for_unmask will not actually unmask, this will
1310	 * be done later by xive_finish_unmask() once the targetting
1311	 * has been done, so we don't try to unmask an interrupt
1312	 * that hasn't yet been targetted.
1313	 */
1314	if (priority == MASKED)
1315		xive_lock_and_mask(xive, sb, state);
1316	else
1317		xive_lock_for_unmask(sb, state);
1318
1319
1320	/*
1321	 * Then we handle targetting.
1322	 *
1323	 * First calculate a new "actual priority"
1324	 */
1325	new_act_prio = state->act_priority;
1326	if (priority != MASKED)
1327		new_act_prio = xive_prio_from_guest(priority);
1328
1329	pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n",
1330		 new_act_prio, state->act_server, state->act_priority);
1331
1332	/*
1333	 * Then check if we actually need to change anything,
1334	 *
1335	 * The condition for re-targetting the interrupt is that
1336	 * we have a valid new priority (new_act_prio is not 0xff)
1337	 * and either the server or the priority changed.
1338	 *
1339	 * Note: If act_priority was ff and the new priority is
1340	 *       also ff, we don't do anything and leave the interrupt
1341	 *       untargetted. An attempt of doing an int_on on an
1342	 *       untargetted interrupt will fail. If that is a problem
1343	 *       we could initialize interrupts with valid default
1344	 */
1345
1346	if (new_act_prio != MASKED &&
1347	    (state->act_server != server ||
1348	     state->act_priority != new_act_prio))
1349		rc = xive_target_interrupt(kvm, state, server, new_act_prio);
1350
1351	/*
1352	 * Perform the final unmasking of the interrupt source
1353	 * if necessary
1354	 */
1355	if (priority != MASKED)
1356		xive_finish_unmask(xive, sb, state, priority);
1357
1358	/*
1359	 * Finally Update saved_priority to match. Only int_on/off
1360	 * set this field to a different value.
1361	 */
1362	state->saved_priority = priority;
1363
1364	arch_spin_unlock(&sb->lock);
1365	return rc;
1366}
1367
1368int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
1369			 u32 *priority)
1370{
1371	struct kvmppc_xive *xive = kvm->arch.xive;
1372	struct kvmppc_xive_src_block *sb;
1373	struct kvmppc_xive_irq_state *state;
1374	u16 idx;
1375
1376	if (!xive)
1377		return -ENODEV;
1378
1379	sb = kvmppc_xive_find_source(xive, irq, &idx);
1380	if (!sb)
1381		return -EINVAL;
1382	state = &sb->irq_state[idx];
1383	arch_spin_lock(&sb->lock);
1384	*server = state->act_server;
1385	*priority = state->guest_priority;
1386	arch_spin_unlock(&sb->lock);
1387
1388	return 0;
1389}
1390
1391int kvmppc_xive_int_on(struct kvm *kvm, u32 irq)
1392{
1393	struct kvmppc_xive *xive = kvm->arch.xive;
1394	struct kvmppc_xive_src_block *sb;
1395	struct kvmppc_xive_irq_state *state;
1396	u16 idx;
1397
1398	if (!xive)
1399		return -ENODEV;
1400
1401	sb = kvmppc_xive_find_source(xive, irq, &idx);
1402	if (!sb)
1403		return -EINVAL;
1404	state = &sb->irq_state[idx];
1405
1406	pr_devel("int_on(irq=0x%x)\n", irq);
1407
1408	/*
1409	 * Check if interrupt was not targetted
1410	 */
1411	if (state->act_priority == MASKED) {
1412		pr_devel("int_on on untargetted interrupt\n");
1413		return -EINVAL;
1414	}
1415
1416	/* If saved_priority is 0xff, do nothing */
1417	if (state->saved_priority == MASKED)
1418		return 0;
1419
1420	/*
1421	 * Lock and unmask it.
1422	 */
1423	xive_lock_for_unmask(sb, state);
1424	xive_finish_unmask(xive, sb, state, state->saved_priority);
1425	arch_spin_unlock(&sb->lock);
1426
1427	return 0;
1428}
1429
1430int kvmppc_xive_int_off(struct kvm *kvm, u32 irq)
1431{
1432	struct kvmppc_xive *xive = kvm->arch.xive;
1433	struct kvmppc_xive_src_block *sb;
1434	struct kvmppc_xive_irq_state *state;
1435	u16 idx;
1436
1437	if (!xive)
1438		return -ENODEV;
1439
1440	sb = kvmppc_xive_find_source(xive, irq, &idx);
1441	if (!sb)
1442		return -EINVAL;
1443	state = &sb->irq_state[idx];
1444
1445	pr_devel("int_off(irq=0x%x)\n", irq);
1446
1447	/*
1448	 * Lock and mask
1449	 */
1450	state->saved_priority = xive_lock_and_mask(xive, sb, state);
1451	arch_spin_unlock(&sb->lock);
1452
1453	return 0;
1454}
1455
1456static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq)
1457{
1458	struct kvmppc_xive_src_block *sb;
1459	struct kvmppc_xive_irq_state *state;
1460	u16 idx;
1461
1462	sb = kvmppc_xive_find_source(xive, irq, &idx);
1463	if (!sb)
1464		return false;
1465	state = &sb->irq_state[idx];
1466	if (!state->valid)
1467		return false;
1468
1469	/*
1470	 * Trigger the IPI. This assumes we never restore a pass-through
1471	 * interrupt which should be safe enough
1472	 */
1473	xive_irq_trigger(&state->ipi_data);
1474
1475	return true;
1476}
1477
1478u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu)
1479{
1480	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1481
1482	if (!xc)
1483		return 0;
1484
1485	/* Return the per-cpu state for state saving/migration */
1486	return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT |
1487	       (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT |
1488	       (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT;
1489}
1490
1491int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval)
1492{
1493	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1494	struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1495	u8 cppr, mfrr;
1496	u32 xisr;
1497
1498	if (!xc || !xive)
1499		return -ENOENT;
1500
1501	/* Grab individual state fields. We don't use pending_pri */
1502	cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT;
1503	xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) &
1504		KVM_REG_PPC_ICP_XISR_MASK;
1505	mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT;
1506
1507	pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n",
1508		 xc->server_num, cppr, mfrr, xisr);
1509
1510	/*
1511	 * We can't update the state of a "pushed" VCPU, but that
1512	 * shouldn't happen because the vcpu->mutex makes running a
1513	 * vcpu mutually exclusive with doing one_reg get/set on it.
1514	 */
1515	if (WARN_ON(vcpu->arch.xive_pushed))
1516		return -EIO;
1517
1518	/* Update VCPU HW saved state */
1519	vcpu->arch.xive_saved_state.cppr = cppr;
1520	xc->hw_cppr = xc->cppr = cppr;
1521
1522	/*
1523	 * Update MFRR state. If it's not 0xff, we mark the VCPU as
1524	 * having a pending MFRR change, which will re-evaluate the
1525	 * target. The VCPU will thus potentially get a spurious
1526	 * interrupt but that's not a big deal.
1527	 */
1528	xc->mfrr = mfrr;
1529	if (mfrr < cppr)
1530		xive_irq_trigger(&xc->vp_ipi_data);
1531
1532	/*
1533	 * Now saved XIRR is "interesting". It means there's something in
1534	 * the legacy "1 element" queue... for an IPI we simply ignore it,
1535	 * as the MFRR restore will handle that. For anything else we need
1536	 * to force a resend of the source.
1537	 * However the source may not have been setup yet. If that's the
1538	 * case, we keep that info and increment a counter in the xive to
1539	 * tell subsequent xive_set_source() to go look.
1540	 */
1541	if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) {
1542		xc->delayed_irq = xisr;
1543		xive->delayed_irqs++;
1544		pr_devel("  xisr restore delayed\n");
1545	}
1546
1547	return 0;
1548}
1549
1550int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
1551			   unsigned long host_irq)
1552{
1553	struct kvmppc_xive *xive = kvm->arch.xive;
1554	struct kvmppc_xive_src_block *sb;
1555	struct kvmppc_xive_irq_state *state;
1556	struct irq_data *host_data =
1557		irq_domain_get_irq_data(irq_get_default_domain(), host_irq);
1558	unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data);
1559	u16 idx;
1560	u8 prio;
1561	int rc;
1562
1563	if (!xive)
1564		return -ENODEV;
1565
1566	pr_debug("%s: GIRQ 0x%lx host IRQ %ld XIVE HW IRQ 0x%x\n",
1567		 __func__, guest_irq, host_irq, hw_irq);
1568
1569	sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
1570	if (!sb)
1571		return -EINVAL;
1572	state = &sb->irq_state[idx];
1573
1574	/*
1575	 * Mark the passed-through interrupt as going to a VCPU,
1576	 * this will prevent further EOIs and similar operations
1577	 * from the XIVE code. It will also mask the interrupt
1578	 * to either PQ=10 or 11 state, the latter if the interrupt
1579	 * is pending. This will allow us to unmask or retrigger it
1580	 * after routing it to the guest with a simple EOI.
1581	 *
1582	 * The "state" argument is a "token", all it needs is to be
1583	 * non-NULL to switch to passed-through or NULL for the
1584	 * other way around. We may not yet have an actual VCPU
1585	 * target here and we don't really care.
1586	 */
1587	rc = irq_set_vcpu_affinity(host_irq, state);
1588	if (rc) {
1589		pr_err("Failed to set VCPU affinity for host IRQ %ld\n", host_irq);
1590		return rc;
1591	}
1592
1593	/*
1594	 * Mask and read state of IPI. We need to know if its P bit
1595	 * is set as that means it's potentially already using a
1596	 * queue entry in the target
1597	 */
1598	prio = xive_lock_and_mask(xive, sb, state);
1599	pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio,
1600		 state->old_p, state->old_q);
1601
1602	/* Turn the IPI hard off */
1603	xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
1604
1605	/*
1606	 * Reset ESB guest mapping. Needed when ESB pages are exposed
1607	 * to the guest in XIVE native mode
1608	 */
1609	if (xive->ops && xive->ops->reset_mapped)
1610		xive->ops->reset_mapped(kvm, guest_irq);
1611
1612	/* Grab info about irq */
1613	state->pt_number = hw_irq;
1614	state->pt_data = irq_data_get_irq_chip_data(host_data);
1615
1616	/*
1617	 * Configure the IRQ to match the existing configuration of
1618	 * the IPI if it was already targetted. Otherwise this will
1619	 * mask the interrupt in a lossy way (act_priority is 0xff)
1620	 * which is fine for a never started interrupt.
1621	 */
1622	xive_native_configure_irq(hw_irq,
1623				  kvmppc_xive_vp(xive, state->act_server),
1624				  state->act_priority, state->number);
1625
1626	/*
1627	 * We do an EOI to enable the interrupt (and retrigger if needed)
1628	 * if the guest has the interrupt unmasked and the P bit was *not*
1629	 * set in the IPI. If it was set, we know a slot may still be in
1630	 * use in the target queue thus we have to wait for a guest
1631	 * originated EOI
1632	 */
1633	if (prio != MASKED && !state->old_p)
1634		xive_vm_source_eoi(hw_irq, state->pt_data);
1635
1636	/* Clear old_p/old_q as they are no longer relevant */
1637	state->old_p = state->old_q = false;
1638
1639	/* Restore guest prio (unlocks EOI) */
1640	mb();
1641	state->guest_priority = prio;
1642	arch_spin_unlock(&sb->lock);
1643
1644	return 0;
1645}
1646EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped);
1647
1648int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
1649			   unsigned long host_irq)
1650{
1651	struct kvmppc_xive *xive = kvm->arch.xive;
1652	struct kvmppc_xive_src_block *sb;
1653	struct kvmppc_xive_irq_state *state;
1654	u16 idx;
1655	u8 prio;
1656	int rc;
1657
1658	if (!xive)
1659		return -ENODEV;
1660
1661	pr_debug("%s: GIRQ 0x%lx host IRQ %ld\n", __func__, guest_irq, host_irq);
1662
1663	sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
1664	if (!sb)
1665		return -EINVAL;
1666	state = &sb->irq_state[idx];
1667
1668	/*
1669	 * Mask and read state of IRQ. We need to know if its P bit
1670	 * is set as that means it's potentially already using a
1671	 * queue entry in the target
1672	 */
1673	prio = xive_lock_and_mask(xive, sb, state);
1674	pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio,
1675		 state->old_p, state->old_q);
1676
1677	/*
1678	 * If old_p is set, the interrupt is pending, we switch it to
1679	 * PQ=11. This will force a resend in the host so the interrupt
1680	 * isn't lost to whatever host driver may pick it up
1681	 */
1682	if (state->old_p)
1683		xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11);
1684
1685	/* Release the passed-through interrupt to the host */
1686	rc = irq_set_vcpu_affinity(host_irq, NULL);
1687	if (rc) {
1688		pr_err("Failed to clr VCPU affinity for host IRQ %ld\n", host_irq);
1689		return rc;
1690	}
1691
1692	/* Forget about the IRQ */
1693	state->pt_number = 0;
1694	state->pt_data = NULL;
1695
1696	/*
1697	 * Reset ESB guest mapping. Needed when ESB pages are exposed
1698	 * to the guest in XIVE native mode
1699	 */
1700	if (xive->ops && xive->ops->reset_mapped) {
1701		xive->ops->reset_mapped(kvm, guest_irq);
1702	}
1703
1704	/* Reconfigure the IPI */
1705	xive_native_configure_irq(state->ipi_number,
1706				  kvmppc_xive_vp(xive, state->act_server),
1707				  state->act_priority, state->number);
1708
1709	/*
1710	 * If old_p is set (we have a queue entry potentially
1711	 * occupied) or the interrupt is masked, we set the IPI
1712	 * to PQ=10 state. Otherwise we just re-enable it (PQ=00).
1713	 */
1714	if (prio == MASKED || state->old_p)
1715		xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10);
1716	else
1717		xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00);
1718
1719	/* Restore guest prio (unlocks EOI) */
1720	mb();
1721	state->guest_priority = prio;
1722	arch_spin_unlock(&sb->lock);
1723
1724	return 0;
1725}
1726EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped);
1727
1728void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu)
1729{
1730	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1731	struct kvm *kvm = vcpu->kvm;
1732	struct kvmppc_xive *xive = kvm->arch.xive;
1733	int i, j;
1734
1735	for (i = 0; i <= xive->max_sbid; i++) {
1736		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1737
1738		if (!sb)
1739			continue;
1740		for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
1741			struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
1742
1743			if (!state->valid)
1744				continue;
1745			if (state->act_priority == MASKED)
1746				continue;
1747			if (state->act_server != xc->server_num)
1748				continue;
1749
1750			/* Clean it up */
1751			arch_spin_lock(&sb->lock);
1752			state->act_priority = MASKED;
1753			xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
1754			xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
1755			if (state->pt_number) {
1756				xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
1757				xive_native_configure_irq(state->pt_number, 0, MASKED, 0);
1758			}
1759			arch_spin_unlock(&sb->lock);
1760		}
1761	}
1762
1763	/* Disable vcpu's escalation interrupt */
1764	if (vcpu->arch.xive_esc_on) {
1765		__raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
1766					     XIVE_ESB_SET_PQ_01));
1767		vcpu->arch.xive_esc_on = false;
1768	}
1769
1770	/*
1771	 * Clear pointers to escalation interrupt ESB.
1772	 * This is safe because the vcpu->mutex is held, preventing
1773	 * any other CPU from concurrently executing a KVM_RUN ioctl.
1774	 */
1775	vcpu->arch.xive_esc_vaddr = 0;
1776	vcpu->arch.xive_esc_raddr = 0;
1777}
1778
1779/*
1780 * In single escalation mode, the escalation interrupt is marked so
1781 * that EOI doesn't re-enable it, but just sets the stale_p flag to
1782 * indicate that the P bit has already been dealt with.  However, the
1783 * assembly code that enters the guest sets PQ to 00 without clearing
1784 * stale_p (because it has no easy way to address it).  Hence we have
1785 * to adjust stale_p before shutting down the interrupt.
1786 */
1787void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu, int irq)
1788{
1789	struct xive_irq_data *xd = irq_get_chip_data(irq);
1790
1791	/*
1792	 * This slightly odd sequence gives the right result
1793	 * (i.e. stale_p set if xive_esc_on is false) even if
1794	 * we race with xive_esc_irq() and xive_irq_eoi().
1795	 */
1796	xd->stale_p = false;
1797	smp_mb();		/* paired with smb_wmb in xive_esc_irq */
1798	if (!vcpu->arch.xive_esc_on)
1799		xd->stale_p = true;
1800}
1801
1802void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)
1803{
1804	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1805	struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1806	int i;
1807
1808	if (!kvmppc_xics_enabled(vcpu))
1809		return;
1810
1811	if (!xc)
1812		return;
1813
1814	pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num);
1815
1816	/* Ensure no interrupt is still routed to that VP */
1817	xc->valid = false;
1818	kvmppc_xive_disable_vcpu_interrupts(vcpu);
1819
1820	/* Mask the VP IPI */
1821	xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);
1822
1823	/* Free escalations */
1824	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1825		if (xc->esc_virq[i]) {
1826			if (kvmppc_xive_has_single_escalation(xc->xive))
1827				xive_cleanup_single_escalation(vcpu, xc->esc_virq[i]);
1828			free_irq(xc->esc_virq[i], vcpu);
1829			irq_dispose_mapping(xc->esc_virq[i]);
1830			kfree(xc->esc_virq_names[i]);
1831		}
1832	}
1833
1834	/* Disable the VP */
1835	xive_native_disable_vp(xc->vp_id);
1836
1837	/* Clear the cam word so guest entry won't try to push context */
1838	vcpu->arch.xive_cam_word = 0;
1839
1840	/* Free the queues */
1841	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1842		struct xive_q *q = &xc->queues[i];
1843
1844		xive_native_disable_queue(xc->vp_id, q, i);
1845		if (q->qpage) {
1846			free_pages((unsigned long)q->qpage,
1847				   xive->q_page_order);
1848			q->qpage = NULL;
1849		}
1850	}
1851
1852	/* Free the IPI */
1853	if (xc->vp_ipi) {
1854		xive_cleanup_irq_data(&xc->vp_ipi_data);
1855		xive_native_free_irq(xc->vp_ipi);
1856	}
1857	/* Free the VP */
1858	kfree(xc);
1859
1860	/* Cleanup the vcpu */
1861	vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
1862	vcpu->arch.xive_vcpu = NULL;
1863}
1864
1865static bool kvmppc_xive_vcpu_id_valid(struct kvmppc_xive *xive, u32 cpu)
1866{
1867	/* We have a block of xive->nr_servers VPs. We just need to check
1868	 * packed vCPU ids are below that.
1869	 */
1870	return kvmppc_pack_vcpu_id(xive->kvm, cpu) < xive->nr_servers;
1871}
1872
1873int kvmppc_xive_compute_vp_id(struct kvmppc_xive *xive, u32 cpu, u32 *vp)
1874{
1875	u32 vp_id;
1876
1877	if (!kvmppc_xive_vcpu_id_valid(xive, cpu)) {
1878		pr_devel("Out of bounds !\n");
1879		return -EINVAL;
1880	}
1881
1882	if (xive->vp_base == XIVE_INVALID_VP) {
1883		xive->vp_base = xive_native_alloc_vp_block(xive->nr_servers);
1884		pr_devel("VP_Base=%x nr_servers=%d\n", xive->vp_base, xive->nr_servers);
1885
1886		if (xive->vp_base == XIVE_INVALID_VP)
1887			return -ENOSPC;
1888	}
1889
1890	vp_id = kvmppc_xive_vp(xive, cpu);
1891	if (kvmppc_xive_vp_in_use(xive->kvm, vp_id)) {
1892		pr_devel("Duplicate !\n");
1893		return -EEXIST;
1894	}
1895
1896	*vp = vp_id;
1897
1898	return 0;
1899}
1900
1901int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
1902			     struct kvm_vcpu *vcpu, u32 cpu)
1903{
1904	struct kvmppc_xive *xive = dev->private;
1905	struct kvmppc_xive_vcpu *xc;
1906	int i, r = -EBUSY;
1907	u32 vp_id;
1908
1909	pr_devel("connect_vcpu(cpu=%d)\n", cpu);
1910
1911	if (dev->ops != &kvm_xive_ops) {
1912		pr_devel("Wrong ops !\n");
1913		return -EPERM;
1914	}
1915	if (xive->kvm != vcpu->kvm)
1916		return -EPERM;
1917	if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT)
1918		return -EBUSY;
1919
1920	/* We need to synchronize with queue provisioning */
1921	mutex_lock(&xive->lock);
1922
1923	r = kvmppc_xive_compute_vp_id(xive, cpu, &vp_id);
1924	if (r)
1925		goto bail;
1926
1927	xc = kzalloc(sizeof(*xc), GFP_KERNEL);
1928	if (!xc) {
1929		r = -ENOMEM;
1930		goto bail;
1931	}
1932
1933	vcpu->arch.xive_vcpu = xc;
1934	xc->xive = xive;
1935	xc->vcpu = vcpu;
1936	xc->server_num = cpu;
1937	xc->vp_id = vp_id;
1938	xc->mfrr = 0xff;
1939	xc->valid = true;
1940
1941	r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
1942	if (r)
1943		goto bail;
1944
1945	if (!kvmppc_xive_check_save_restore(vcpu)) {
1946		pr_err("inconsistent save-restore setup for VCPU %d\n", cpu);
1947		r = -EIO;
1948		goto bail;
1949	}
1950
1951	/* Configure VCPU fields for use by assembly push/pull */
1952	vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
1953	vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
1954
1955	/* Allocate IPI */
1956	xc->vp_ipi = xive_native_alloc_irq();
1957	if (!xc->vp_ipi) {
1958		pr_err("Failed to allocate xive irq for VCPU IPI\n");
1959		r = -EIO;
1960		goto bail;
1961	}
1962	pr_devel(" IPI=0x%x\n", xc->vp_ipi);
1963
1964	r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data);
1965	if (r)
1966		goto bail;
1967
1968	/*
1969	 * Enable the VP first as the single escalation mode will
1970	 * affect escalation interrupts numbering
1971	 */
1972	r = xive_native_enable_vp(xc->vp_id, kvmppc_xive_has_single_escalation(xive));
1973	if (r) {
1974		pr_err("Failed to enable VP in OPAL, err %d\n", r);
1975		goto bail;
1976	}
1977
1978	/*
1979	 * Initialize queues. Initially we set them all for no queueing
1980	 * and we enable escalation for queue 0 only which we'll use for
1981	 * our mfrr change notifications. If the VCPU is hot-plugged, we
1982	 * do handle provisioning however based on the existing "map"
1983	 * of enabled queues.
1984	 */
1985	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1986		struct xive_q *q = &xc->queues[i];
1987
1988		/* Single escalation, no queue 7 */
1989		if (i == 7 && kvmppc_xive_has_single_escalation(xive))
1990			break;
1991
1992		/* Is queue already enabled ? Provision it */
1993		if (xive->qmap & (1 << i)) {
1994			r = xive_provision_queue(vcpu, i);
1995			if (r == 0 && !kvmppc_xive_has_single_escalation(xive))
1996				kvmppc_xive_attach_escalation(
1997					vcpu, i, kvmppc_xive_has_single_escalation(xive));
1998			if (r)
1999				goto bail;
2000		} else {
2001			r = xive_native_configure_queue(xc->vp_id,
2002							q, i, NULL, 0, true);
2003			if (r) {
2004				pr_err("Failed to configure queue %d for VCPU %d\n",
2005				       i, cpu);
2006				goto bail;
2007			}
2008		}
2009	}
2010
2011	/* If not done above, attach priority 0 escalation */
2012	r = kvmppc_xive_attach_escalation(vcpu, 0, kvmppc_xive_has_single_escalation(xive));
2013	if (r)
2014		goto bail;
2015
2016	/* Route the IPI */
2017	r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI);
2018	if (!r)
2019		xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00);
2020
2021bail:
2022	mutex_unlock(&xive->lock);
2023	if (r) {
2024		kvmppc_xive_cleanup_vcpu(vcpu);
2025		return r;
2026	}
2027
2028	vcpu->arch.irq_type = KVMPPC_IRQ_XICS;
2029	return 0;
2030}
2031
2032/*
2033 * Scanning of queues before/after migration save
2034 */
2035static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq)
2036{
2037	struct kvmppc_xive_src_block *sb;
2038	struct kvmppc_xive_irq_state *state;
2039	u16 idx;
2040
2041	sb = kvmppc_xive_find_source(xive, irq, &idx);
2042	if (!sb)
2043		return;
2044
2045	state = &sb->irq_state[idx];
2046
2047	/* Some sanity checking */
2048	if (!state->valid) {
2049		pr_err("invalid irq 0x%x in cpu queue!\n", irq);
2050		return;
2051	}
2052
2053	/*
2054	 * If the interrupt is in a queue it should have P set.
2055	 * We warn so that gets reported. A backtrace isn't useful
2056	 * so no need to use a WARN_ON.
2057	 */
2058	if (!state->saved_p)
2059		pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq);
2060
2061	/* Set flag */
2062	state->in_queue = true;
2063}
2064
2065static void xive_pre_save_mask_irq(struct kvmppc_xive *xive,
2066				   struct kvmppc_xive_src_block *sb,
2067				   u32 irq)
2068{
2069	struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
2070
2071	if (!state->valid)
2072		return;
2073
2074	/* Mask and save state, this will also sync HW queues */
2075	state->saved_scan_prio = xive_lock_and_mask(xive, sb, state);
2076
2077	/* Transfer P and Q */
2078	state->saved_p = state->old_p;
2079	state->saved_q = state->old_q;
2080
2081	/* Unlock */
2082	arch_spin_unlock(&sb->lock);
2083}
2084
2085static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive,
2086				     struct kvmppc_xive_src_block *sb,
2087				     u32 irq)
2088{
2089	struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
2090
2091	if (!state->valid)
2092		return;
2093
2094	/*
2095	 * Lock / exclude EOI (not technically necessary if the
2096	 * guest isn't running concurrently. If this becomes a
2097	 * performance issue we can probably remove the lock.
2098	 */
2099	xive_lock_for_unmask(sb, state);
2100
2101	/* Restore mask/prio if it wasn't masked */
2102	if (state->saved_scan_prio != MASKED)
2103		xive_finish_unmask(xive, sb, state, state->saved_scan_prio);
2104
2105	/* Unlock */
2106	arch_spin_unlock(&sb->lock);
2107}
2108
2109static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q)
2110{
2111	u32 idx = q->idx;
2112	u32 toggle = q->toggle;
2113	u32 irq;
2114
2115	do {
2116		irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle);
2117		if (irq > XICS_IPI)
2118			xive_pre_save_set_queued(xive, irq);
2119	} while(irq);
2120}
2121
2122static void xive_pre_save_scan(struct kvmppc_xive *xive)
2123{
2124	struct kvm_vcpu *vcpu = NULL;
2125	unsigned long i;
2126	int j;
2127
2128	/*
2129	 * See comment in xive_get_source() about how this
2130	 * work. Collect a stable state for all interrupts
2131	 */
2132	for (i = 0; i <= xive->max_sbid; i++) {
2133		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2134		if (!sb)
2135			continue;
2136		for (j = 0;  j < KVMPPC_XICS_IRQ_PER_ICS; j++)
2137			xive_pre_save_mask_irq(xive, sb, j);
2138	}
2139
2140	/* Then scan the queues and update the "in_queue" flag */
2141	kvm_for_each_vcpu(i, vcpu, xive->kvm) {
2142		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2143		if (!xc)
2144			continue;
2145		for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) {
2146			if (xc->queues[j].qpage)
2147				xive_pre_save_queue(xive, &xc->queues[j]);
2148		}
2149	}
2150
2151	/* Finally restore interrupt states */
2152	for (i = 0; i <= xive->max_sbid; i++) {
2153		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2154		if (!sb)
2155			continue;
2156		for (j = 0;  j < KVMPPC_XICS_IRQ_PER_ICS; j++)
2157			xive_pre_save_unmask_irq(xive, sb, j);
2158	}
2159}
2160
2161static void xive_post_save_scan(struct kvmppc_xive *xive)
2162{
2163	u32 i, j;
2164
2165	/* Clear all the in_queue flags */
2166	for (i = 0; i <= xive->max_sbid; i++) {
2167		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2168		if (!sb)
2169			continue;
2170		for (j = 0;  j < KVMPPC_XICS_IRQ_PER_ICS; j++)
2171			sb->irq_state[j].in_queue = false;
2172	}
2173
2174	/* Next get_source() will do a new scan */
2175	xive->saved_src_count = 0;
2176}
2177
2178/*
2179 * This returns the source configuration and state to user space.
2180 */
2181static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr)
2182{
2183	struct kvmppc_xive_src_block *sb;
2184	struct kvmppc_xive_irq_state *state;
2185	u64 __user *ubufp = (u64 __user *) addr;
2186	u64 val, prio;
2187	u16 idx;
2188
2189	sb = kvmppc_xive_find_source(xive, irq, &idx);
2190	if (!sb)
2191		return -ENOENT;
2192
2193	state = &sb->irq_state[idx];
2194
2195	if (!state->valid)
2196		return -ENOENT;
2197
2198	pr_devel("get_source(%ld)...\n", irq);
2199
2200	/*
2201	 * So to properly save the state into something that looks like a
2202	 * XICS migration stream we cannot treat interrupts individually.
2203	 *
2204	 * We need, instead, mask them all (& save their previous PQ state)
2205	 * to get a stable state in the HW, then sync them to ensure that
2206	 * any interrupt that had already fired hits its queue, and finally
2207	 * scan all the queues to collect which interrupts are still present
2208	 * in the queues, so we can set the "pending" flag on them and
2209	 * they can be resent on restore.
2210	 *
2211	 * So we do it all when the "first" interrupt gets saved, all the
2212	 * state is collected at that point, the rest of xive_get_source()
2213	 * will merely collect and convert that state to the expected
2214	 * userspace bit mask.
2215	 */
2216	if (xive->saved_src_count == 0)
2217		xive_pre_save_scan(xive);
2218	xive->saved_src_count++;
2219
2220	/* Convert saved state into something compatible with xics */
2221	val = state->act_server;
2222	prio = state->saved_scan_prio;
2223
2224	if (prio == MASKED) {
2225		val |= KVM_XICS_MASKED;
2226		prio = state->saved_priority;
2227	}
2228	val |= prio << KVM_XICS_PRIORITY_SHIFT;
2229	if (state->lsi) {
2230		val |= KVM_XICS_LEVEL_SENSITIVE;
2231		if (state->saved_p)
2232			val |= KVM_XICS_PENDING;
2233	} else {
2234		if (state->saved_p)
2235			val |= KVM_XICS_PRESENTED;
2236
2237		if (state->saved_q)
2238			val |= KVM_XICS_QUEUED;
2239
2240		/*
2241		 * We mark it pending (which will attempt a re-delivery)
2242		 * if we are in a queue *or* we were masked and had
2243		 * Q set which is equivalent to the XICS "masked pending"
2244		 * state
2245		 */
2246		if (state->in_queue || (prio == MASKED && state->saved_q))
2247			val |= KVM_XICS_PENDING;
2248	}
2249
2250	/*
2251	 * If that was the last interrupt saved, reset the
2252	 * in_queue flags
2253	 */
2254	if (xive->saved_src_count == xive->src_count)
2255		xive_post_save_scan(xive);
2256
2257	/* Copy the result to userspace */
2258	if (put_user(val, ubufp))
2259		return -EFAULT;
2260
2261	return 0;
2262}
2263
2264struct kvmppc_xive_src_block *kvmppc_xive_create_src_block(
2265	struct kvmppc_xive *xive, int irq)
2266{
2267	struct kvmppc_xive_src_block *sb;
2268	int i, bid;
2269
2270	bid = irq >> KVMPPC_XICS_ICS_SHIFT;
2271
2272	mutex_lock(&xive->lock);
2273
2274	/* block already exists - somebody else got here first */
2275	if (xive->src_blocks[bid])
2276		goto out;
2277
2278	/* Create the ICS */
2279	sb = kzalloc(sizeof(*sb), GFP_KERNEL);
2280	if (!sb)
2281		goto out;
2282
2283	sb->id = bid;
2284
2285	for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2286		sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i;
2287		sb->irq_state[i].eisn = 0;
2288		sb->irq_state[i].guest_priority = MASKED;
2289		sb->irq_state[i].saved_priority = MASKED;
2290		sb->irq_state[i].act_priority = MASKED;
2291	}
2292	smp_wmb();
2293	xive->src_blocks[bid] = sb;
2294
2295	if (bid > xive->max_sbid)
2296		xive->max_sbid = bid;
2297
2298out:
2299	mutex_unlock(&xive->lock);
2300	return xive->src_blocks[bid];
2301}
2302
2303static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq)
2304{
2305	struct kvm *kvm = xive->kvm;
2306	struct kvm_vcpu *vcpu = NULL;
2307	unsigned long i;
2308
2309	kvm_for_each_vcpu(i, vcpu, kvm) {
2310		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2311
2312		if (!xc)
2313			continue;
2314
2315		if (xc->delayed_irq == irq) {
2316			xc->delayed_irq = 0;
2317			xive->delayed_irqs--;
2318			return true;
2319		}
2320	}
2321	return false;
2322}
2323
2324static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr)
2325{
2326	struct kvmppc_xive_src_block *sb;
2327	struct kvmppc_xive_irq_state *state;
2328	u64 __user *ubufp = (u64 __user *) addr;
2329	u16 idx;
2330	u64 val;
2331	u8 act_prio, guest_prio;
2332	u32 server;
2333	int rc = 0;
2334
2335	if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS)
2336		return -ENOENT;
2337
2338	pr_devel("set_source(irq=0x%lx)\n", irq);
2339
2340	/* Find the source */
2341	sb = kvmppc_xive_find_source(xive, irq, &idx);
2342	if (!sb) {
2343		pr_devel("No source, creating source block...\n");
2344		sb = kvmppc_xive_create_src_block(xive, irq);
2345		if (!sb) {
2346			pr_devel("Failed to create block...\n");
2347			return -ENOMEM;
2348		}
2349	}
2350	state = &sb->irq_state[idx];
2351
2352	/* Read user passed data */
2353	if (get_user(val, ubufp)) {
2354		pr_devel("fault getting user info !\n");
2355		return -EFAULT;
2356	}
2357
2358	server = val & KVM_XICS_DESTINATION_MASK;
2359	guest_prio = val >> KVM_XICS_PRIORITY_SHIFT;
2360
2361	pr_devel("  val=0x016%llx (server=0x%x, guest_prio=%d)\n",
2362		 val, server, guest_prio);
2363
2364	/*
2365	 * If the source doesn't already have an IPI, allocate
2366	 * one and get the corresponding data
2367	 */
2368	if (!state->ipi_number) {
2369		state->ipi_number = xive_native_alloc_irq();
2370		if (state->ipi_number == 0) {
2371			pr_devel("Failed to allocate IPI !\n");
2372			return -ENOMEM;
2373		}
2374		xive_native_populate_irq_data(state->ipi_number, &state->ipi_data);
2375		pr_devel(" src_ipi=0x%x\n", state->ipi_number);
2376	}
2377
2378	/*
2379	 * We use lock_and_mask() to set us in the right masked
2380	 * state. We will override that state from the saved state
2381	 * further down, but this will handle the cases of interrupts
2382	 * that need FW masking. We set the initial guest_priority to
2383	 * 0 before calling it to ensure it actually performs the masking.
2384	 */
2385	state->guest_priority = 0;
2386	xive_lock_and_mask(xive, sb, state);
2387
2388	/*
2389	 * Now, we select a target if we have one. If we don't we
2390	 * leave the interrupt untargetted. It means that an interrupt
2391	 * can become "untargetted" across migration if it was masked
2392	 * by set_xive() but there is little we can do about it.
2393	 */
2394
2395	/* First convert prio and mark interrupt as untargetted */
2396	act_prio = xive_prio_from_guest(guest_prio);
2397	state->act_priority = MASKED;
2398
2399	/*
2400	 * We need to drop the lock due to the mutex below. Hopefully
2401	 * nothing is touching that interrupt yet since it hasn't been
2402	 * advertized to a running guest yet
2403	 */
2404	arch_spin_unlock(&sb->lock);
2405
2406	/* If we have a priority target the interrupt */
2407	if (act_prio != MASKED) {
2408		/* First, check provisioning of queues */
2409		mutex_lock(&xive->lock);
2410		rc = xive_check_provisioning(xive->kvm, act_prio);
2411		mutex_unlock(&xive->lock);
2412
2413		/* Target interrupt */
2414		if (rc == 0)
2415			rc = xive_target_interrupt(xive->kvm, state,
2416						   server, act_prio);
2417		/*
2418		 * If provisioning or targetting failed, leave it
2419		 * alone and masked. It will remain disabled until
2420		 * the guest re-targets it.
2421		 */
2422	}
2423
2424	/*
2425	 * Find out if this was a delayed irq stashed in an ICP,
2426	 * in which case, treat it as pending
2427	 */
2428	if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) {
2429		val |= KVM_XICS_PENDING;
2430		pr_devel("  Found delayed ! forcing PENDING !\n");
2431	}
2432
2433	/* Cleanup the SW state */
2434	state->old_p = false;
2435	state->old_q = false;
2436	state->lsi = false;
2437	state->asserted = false;
2438
2439	/* Restore LSI state */
2440	if (val & KVM_XICS_LEVEL_SENSITIVE) {
2441		state->lsi = true;
2442		if (val & KVM_XICS_PENDING)
2443			state->asserted = true;
2444		pr_devel("  LSI ! Asserted=%d\n", state->asserted);
2445	}
2446
2447	/*
2448	 * Restore P and Q. If the interrupt was pending, we
2449	 * force Q and !P, which will trigger a resend.
2450	 *
2451	 * That means that a guest that had both an interrupt
2452	 * pending (queued) and Q set will restore with only
2453	 * one instance of that interrupt instead of 2, but that
2454	 * is perfectly fine as coalescing interrupts that haven't
2455	 * been presented yet is always allowed.
2456	 */
2457	if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING))
2458		state->old_p = true;
2459	if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING)
2460		state->old_q = true;
2461
2462	pr_devel("  P=%d, Q=%d\n", state->old_p, state->old_q);
2463
2464	/*
2465	 * If the interrupt was unmasked, update guest priority and
2466	 * perform the appropriate state transition and do a
2467	 * re-trigger if necessary.
2468	 */
2469	if (val & KVM_XICS_MASKED) {
2470		pr_devel("  masked, saving prio\n");
2471		state->guest_priority = MASKED;
2472		state->saved_priority = guest_prio;
2473	} else {
2474		pr_devel("  unmasked, restoring to prio %d\n", guest_prio);
2475		xive_finish_unmask(xive, sb, state, guest_prio);
2476		state->saved_priority = guest_prio;
2477	}
2478
2479	/* Increment the number of valid sources and mark this one valid */
2480	if (!state->valid)
2481		xive->src_count++;
2482	state->valid = true;
2483
2484	return 0;
2485}
2486
2487int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
2488			bool line_status)
2489{
2490	struct kvmppc_xive *xive = kvm->arch.xive;
2491	struct kvmppc_xive_src_block *sb;
2492	struct kvmppc_xive_irq_state *state;
2493	u16 idx;
2494
2495	if (!xive)
2496		return -ENODEV;
2497
2498	sb = kvmppc_xive_find_source(xive, irq, &idx);
2499	if (!sb)
2500		return -EINVAL;
2501
2502	/* Perform locklessly .... (we need to do some RCUisms here...) */
2503	state = &sb->irq_state[idx];
2504	if (!state->valid)
2505		return -EINVAL;
2506
2507	/* We don't allow a trigger on a passed-through interrupt */
2508	if (state->pt_number)
2509		return -EINVAL;
2510
2511	if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL)
2512		state->asserted = true;
2513	else if (level == 0 || level == KVM_INTERRUPT_UNSET) {
2514		state->asserted = false;
2515		return 0;
2516	}
2517
2518	/* Trigger the IPI */
2519	xive_irq_trigger(&state->ipi_data);
2520
2521	return 0;
2522}
2523
2524int kvmppc_xive_set_nr_servers(struct kvmppc_xive *xive, u64 addr)
2525{
2526	u32 __user *ubufp = (u32 __user *) addr;
2527	u32 nr_servers;
2528	int rc = 0;
2529
2530	if (get_user(nr_servers, ubufp))
2531		return -EFAULT;
2532
2533	pr_devel("%s nr_servers=%u\n", __func__, nr_servers);
2534
2535	if (!nr_servers || nr_servers > KVM_MAX_VCPU_IDS)
2536		return -EINVAL;
2537
2538	mutex_lock(&xive->lock);
2539	if (xive->vp_base != XIVE_INVALID_VP)
2540		/* The VP block is allocated once and freed when the device
2541		 * is released. Better not allow to change its size since its
2542		 * used by connect_vcpu to validate vCPU ids are valid (eg,
2543		 * setting it back to a higher value could allow connect_vcpu
2544		 * to come up with a VP id that goes beyond the VP block, which
2545		 * is likely to cause a crash in OPAL).
2546		 */
2547		rc = -EBUSY;
2548	else if (nr_servers > KVM_MAX_VCPUS)
2549		/* We don't need more servers. Higher vCPU ids get packed
2550		 * down below KVM_MAX_VCPUS by kvmppc_pack_vcpu_id().
2551		 */
2552		xive->nr_servers = KVM_MAX_VCPUS;
2553	else
2554		xive->nr_servers = nr_servers;
2555
2556	mutex_unlock(&xive->lock);
2557
2558	return rc;
2559}
2560
2561static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2562{
2563	struct kvmppc_xive *xive = dev->private;
2564
2565	/* We honor the existing XICS ioctl */
2566	switch (attr->group) {
2567	case KVM_DEV_XICS_GRP_SOURCES:
2568		return xive_set_source(xive, attr->attr, attr->addr);
2569	case KVM_DEV_XICS_GRP_CTRL:
2570		switch (attr->attr) {
2571		case KVM_DEV_XICS_NR_SERVERS:
2572			return kvmppc_xive_set_nr_servers(xive, attr->addr);
2573		}
2574	}
2575	return -ENXIO;
2576}
2577
2578static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2579{
2580	struct kvmppc_xive *xive = dev->private;
2581
2582	/* We honor the existing XICS ioctl */
2583	switch (attr->group) {
2584	case KVM_DEV_XICS_GRP_SOURCES:
2585		return xive_get_source(xive, attr->attr, attr->addr);
2586	}
2587	return -ENXIO;
2588}
2589
2590static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2591{
2592	/* We honor the same limits as XICS, at least for now */
2593	switch (attr->group) {
2594	case KVM_DEV_XICS_GRP_SOURCES:
2595		if (attr->attr >= KVMPPC_XICS_FIRST_IRQ &&
2596		    attr->attr < KVMPPC_XICS_NR_IRQS)
2597			return 0;
2598		break;
2599	case KVM_DEV_XICS_GRP_CTRL:
2600		switch (attr->attr) {
2601		case KVM_DEV_XICS_NR_SERVERS:
2602			return 0;
2603		}
2604	}
2605	return -ENXIO;
2606}
2607
2608static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd)
2609{
2610	xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
2611	xive_native_configure_irq(hw_num, 0, MASKED, 0);
2612}
2613
2614void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb)
2615{
2616	int i;
2617
2618	for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2619		struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
2620
2621		if (!state->valid)
2622			continue;
2623
2624		kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data);
2625		xive_cleanup_irq_data(&state->ipi_data);
2626		xive_native_free_irq(state->ipi_number);
2627
2628		/* Pass-through, cleanup too but keep IRQ hw data */
2629		if (state->pt_number)
2630			kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data);
2631
2632		state->valid = false;
2633	}
2634}
2635
2636/*
2637 * Called when device fd is closed.  kvm->lock is held.
2638 */
2639static void kvmppc_xive_release(struct kvm_device *dev)
2640{
2641	struct kvmppc_xive *xive = dev->private;
2642	struct kvm *kvm = xive->kvm;
2643	struct kvm_vcpu *vcpu;
2644	unsigned long i;
2645
2646	pr_devel("Releasing xive device\n");
2647
2648	/*
2649	 * Since this is the device release function, we know that
2650	 * userspace does not have any open fd referring to the
2651	 * device.  Therefore there can not be any of the device
2652	 * attribute set/get functions being executed concurrently,
2653	 * and similarly, the connect_vcpu and set/clr_mapped
2654	 * functions also cannot be being executed.
2655	 */
2656
2657	debugfs_remove(xive->dentry);
2658
2659	/*
2660	 * We should clean up the vCPU interrupt presenters first.
2661	 */
2662	kvm_for_each_vcpu(i, vcpu, kvm) {
2663		/*
2664		 * Take vcpu->mutex to ensure that no one_reg get/set ioctl
2665		 * (i.e. kvmppc_xive_[gs]et_icp) can be done concurrently.
2666		 * Holding the vcpu->mutex also means that the vcpu cannot
2667		 * be executing the KVM_RUN ioctl, and therefore it cannot
2668		 * be executing the XIVE push or pull code or accessing
2669		 * the XIVE MMIO regions.
2670		 */
2671		mutex_lock(&vcpu->mutex);
2672		kvmppc_xive_cleanup_vcpu(vcpu);
2673		mutex_unlock(&vcpu->mutex);
2674	}
2675
2676	/*
2677	 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type
2678	 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe
2679	 * against xive code getting called during vcpu execution or
2680	 * set/get one_reg operations.
2681	 */
2682	kvm->arch.xive = NULL;
2683
2684	/* Mask and free interrupts */
2685	for (i = 0; i <= xive->max_sbid; i++) {
2686		if (xive->src_blocks[i])
2687			kvmppc_xive_free_sources(xive->src_blocks[i]);
2688		kfree(xive->src_blocks[i]);
2689		xive->src_blocks[i] = NULL;
2690	}
2691
2692	if (xive->vp_base != XIVE_INVALID_VP)
2693		xive_native_free_vp_block(xive->vp_base);
2694
2695	/*
2696	 * A reference of the kvmppc_xive pointer is now kept under
2697	 * the xive_devices struct of the machine for reuse. It is
2698	 * freed when the VM is destroyed for now until we fix all the
2699	 * execution paths.
2700	 */
2701
2702	kfree(dev);
2703}
2704
2705/*
2706 * When the guest chooses the interrupt mode (XICS legacy or XIVE
2707 * native), the VM will switch of KVM device. The previous device will
2708 * be "released" before the new one is created.
2709 *
2710 * Until we are sure all execution paths are well protected, provide a
2711 * fail safe (transitional) method for device destruction, in which
2712 * the XIVE device pointer is recycled and not directly freed.
2713 */
2714struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type)
2715{
2716	struct kvmppc_xive **kvm_xive_device = type == KVM_DEV_TYPE_XIVE ?
2717		&kvm->arch.xive_devices.native :
2718		&kvm->arch.xive_devices.xics_on_xive;
2719	struct kvmppc_xive *xive = *kvm_xive_device;
2720
2721	if (!xive) {
2722		xive = kzalloc(sizeof(*xive), GFP_KERNEL);
2723		*kvm_xive_device = xive;
2724	} else {
2725		memset(xive, 0, sizeof(*xive));
2726	}
2727
2728	return xive;
2729}
2730
2731/*
2732 * Create a XICS device with XIVE backend.  kvm->lock is held.
2733 */
2734static int kvmppc_xive_create(struct kvm_device *dev, u32 type)
2735{
2736	struct kvmppc_xive *xive;
2737	struct kvm *kvm = dev->kvm;
2738
2739	pr_devel("Creating xive for partition\n");
2740
2741	/* Already there ? */
2742	if (kvm->arch.xive)
2743		return -EEXIST;
2744
2745	xive = kvmppc_xive_get_device(kvm, type);
2746	if (!xive)
2747		return -ENOMEM;
2748
2749	dev->private = xive;
2750	xive->dev = dev;
2751	xive->kvm = kvm;
2752	mutex_init(&xive->lock);
2753
2754	/* We use the default queue size set by the host */
2755	xive->q_order = xive_native_default_eq_shift();
2756	if (xive->q_order < PAGE_SHIFT)
2757		xive->q_page_order = 0;
2758	else
2759		xive->q_page_order = xive->q_order - PAGE_SHIFT;
2760
2761	/* VP allocation is delayed to the first call to connect_vcpu */
2762	xive->vp_base = XIVE_INVALID_VP;
2763	/* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets
2764	 * on a POWER9 system.
2765	 */
2766	xive->nr_servers = KVM_MAX_VCPUS;
2767
2768	if (xive_native_has_single_escalation())
2769		xive->flags |= KVMPPC_XIVE_FLAG_SINGLE_ESCALATION;
2770
2771	if (xive_native_has_save_restore())
2772		xive->flags |= KVMPPC_XIVE_FLAG_SAVE_RESTORE;
2773
2774	kvm->arch.xive = xive;
2775	return 0;
2776}
2777
2778int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req)
2779{
2780	/* The VM should have configured XICS mode before doing XICS hcalls. */
2781	if (!kvmppc_xics_enabled(vcpu))
2782		return H_TOO_HARD;
2783
2784	switch (req) {
2785	case H_XIRR:
2786		return xive_vm_h_xirr(vcpu);
2787	case H_CPPR:
2788		return xive_vm_h_cppr(vcpu, kvmppc_get_gpr(vcpu, 4));
2789	case H_EOI:
2790		return xive_vm_h_eoi(vcpu, kvmppc_get_gpr(vcpu, 4));
2791	case H_IPI:
2792		return xive_vm_h_ipi(vcpu, kvmppc_get_gpr(vcpu, 4),
2793					  kvmppc_get_gpr(vcpu, 5));
2794	case H_IPOLL:
2795		return xive_vm_h_ipoll(vcpu, kvmppc_get_gpr(vcpu, 4));
2796	case H_XIRR_X:
2797		xive_vm_h_xirr(vcpu);
2798		kvmppc_set_gpr(vcpu, 5, get_tb() + kvmppc_get_tb_offset(vcpu));
2799		return H_SUCCESS;
2800	}
2801
2802	return H_UNSUPPORTED;
2803}
2804EXPORT_SYMBOL_GPL(kvmppc_xive_xics_hcall);
2805
2806int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu)
2807{
2808	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2809	unsigned int i;
2810
2811	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
2812		struct xive_q *q = &xc->queues[i];
2813		u32 i0, i1, idx;
2814
2815		if (!q->qpage && !xc->esc_virq[i])
2816			continue;
2817
2818		if (q->qpage) {
2819			seq_printf(m, "    q[%d]: ", i);
2820			idx = q->idx;
2821			i0 = be32_to_cpup(q->qpage + idx);
2822			idx = (idx + 1) & q->msk;
2823			i1 = be32_to_cpup(q->qpage + idx);
2824			seq_printf(m, "T=%d %08x %08x...\n", q->toggle,
2825				   i0, i1);
2826		}
2827		if (xc->esc_virq[i]) {
2828			struct xive_irq_data *xd = irq_get_chip_data(xc->esc_virq[i]);
2829			u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
2830
2831			seq_printf(m, "    ESC %d %c%c EOI @%llx",
2832				   xc->esc_virq[i],
2833				   (pq & XIVE_ESB_VAL_P) ? 'P' : '-',
2834				   (pq & XIVE_ESB_VAL_Q) ? 'Q' : '-',
2835				   xd->eoi_page);
2836			seq_puts(m, "\n");
2837		}
2838	}
2839	return 0;
2840}
2841
2842void kvmppc_xive_debug_show_sources(struct seq_file *m,
2843				    struct kvmppc_xive_src_block *sb)
2844{
2845	int i;
2846
2847	seq_puts(m, "    LISN      HW/CHIP   TYPE    PQ      EISN    CPU/PRIO\n");
2848	for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2849		struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
2850		struct xive_irq_data *xd;
2851		u64 pq;
2852		u32 hw_num;
2853
2854		if (!state->valid)
2855			continue;
2856
2857		kvmppc_xive_select_irq(state, &hw_num, &xd);
2858
2859		pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
2860
2861		seq_printf(m, "%08x  %08x/%02x", state->number, hw_num,
2862			   xd->src_chip);
2863		if (state->lsi)
2864			seq_printf(m, " %cLSI", state->asserted ? '^' : ' ');
2865		else
2866			seq_puts(m, "  MSI");
2867
2868		seq_printf(m, " %s  %c%c  %08x   % 4d/%d",
2869			   state->ipi_number == hw_num ? "IPI" : " PT",
2870			   pq & XIVE_ESB_VAL_P ? 'P' : '-',
2871			   pq & XIVE_ESB_VAL_Q ? 'Q' : '-',
2872			   state->eisn, state->act_server,
2873			   state->act_priority);
2874
2875		seq_puts(m, "\n");
2876	}
2877}
2878
2879static int xive_debug_show(struct seq_file *m, void *private)
2880{
2881	struct kvmppc_xive *xive = m->private;
2882	struct kvm *kvm = xive->kvm;
2883	struct kvm_vcpu *vcpu;
2884	u64 t_rm_h_xirr = 0;
2885	u64 t_rm_h_ipoll = 0;
2886	u64 t_rm_h_cppr = 0;
2887	u64 t_rm_h_eoi = 0;
2888	u64 t_rm_h_ipi = 0;
2889	u64 t_vm_h_xirr = 0;
2890	u64 t_vm_h_ipoll = 0;
2891	u64 t_vm_h_cppr = 0;
2892	u64 t_vm_h_eoi = 0;
2893	u64 t_vm_h_ipi = 0;
2894	unsigned long i;
2895
2896	if (!kvm)
2897		return 0;
2898
2899	seq_puts(m, "=========\nVCPU state\n=========\n");
2900
2901	kvm_for_each_vcpu(i, vcpu, kvm) {
2902		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2903
2904		if (!xc)
2905			continue;
2906
2907		seq_printf(m, "VCPU %d: VP:%#x/%02x\n"
2908			 "    CPPR:%#x HWCPPR:%#x MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n",
2909			 xc->server_num, xc->vp_id, xc->vp_chip_id,
2910			 xc->cppr, xc->hw_cppr,
2911			 xc->mfrr, xc->pending,
2912			 xc->stat_rm_h_xirr, xc->stat_vm_h_xirr);
2913
2914		kvmppc_xive_debug_show_queues(m, vcpu);
2915
2916		t_rm_h_xirr += xc->stat_rm_h_xirr;
2917		t_rm_h_ipoll += xc->stat_rm_h_ipoll;
2918		t_rm_h_cppr += xc->stat_rm_h_cppr;
2919		t_rm_h_eoi += xc->stat_rm_h_eoi;
2920		t_rm_h_ipi += xc->stat_rm_h_ipi;
2921		t_vm_h_xirr += xc->stat_vm_h_xirr;
2922		t_vm_h_ipoll += xc->stat_vm_h_ipoll;
2923		t_vm_h_cppr += xc->stat_vm_h_cppr;
2924		t_vm_h_eoi += xc->stat_vm_h_eoi;
2925		t_vm_h_ipi += xc->stat_vm_h_ipi;
2926	}
2927
2928	seq_puts(m, "Hcalls totals\n");
2929	seq_printf(m, " H_XIRR  R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr);
2930	seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll);
2931	seq_printf(m, " H_CPPR  R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr);
2932	seq_printf(m, " H_EOI   R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi);
2933	seq_printf(m, " H_IPI   R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi);
2934
2935	seq_puts(m, "=========\nSources\n=========\n");
2936
2937	for (i = 0; i <= xive->max_sbid; i++) {
2938		struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2939
2940		if (sb) {
2941			arch_spin_lock(&sb->lock);
2942			kvmppc_xive_debug_show_sources(m, sb);
2943			arch_spin_unlock(&sb->lock);
2944		}
2945	}
2946
2947	return 0;
2948}
2949
2950DEFINE_SHOW_ATTRIBUTE(xive_debug);
2951
2952static void xive_debugfs_init(struct kvmppc_xive *xive)
2953{
2954	xive->dentry = debugfs_create_file("xive", S_IRUGO, xive->kvm->debugfs_dentry,
2955					   xive, &xive_debug_fops);
2956
2957	pr_debug("%s: created\n", __func__);
2958}
2959
2960static void kvmppc_xive_init(struct kvm_device *dev)
2961{
2962	struct kvmppc_xive *xive = dev->private;
2963
2964	/* Register some debug interfaces */
2965	xive_debugfs_init(xive);
2966}
2967
2968struct kvm_device_ops kvm_xive_ops = {
2969	.name = "kvm-xive",
2970	.create = kvmppc_xive_create,
2971	.init = kvmppc_xive_init,
2972	.release = kvmppc_xive_release,
2973	.set_attr = xive_set_attr,
2974	.get_attr = xive_get_attr,
2975	.has_attr = xive_has_attr,
2976};