irq_64.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* irq.c: UltraSparc IRQ handling/init/registry.
   3 *
   4 * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net)
   5 * Copyright (C) 1998  Eddie C. Dost    (ecd@skynet.be)
   6 * Copyright (C) 1998  Jakub Jelinek    (jj@ultra.linux.cz)
   7 */
   8
   9#include <linux/sched.h>
  10#include <linux/linkage.h>
  11#include <linux/ptrace.h>
  12#include <linux/errno.h>
  13#include <linux/kernel_stat.h>
  14#include <linux/signal.h>
  15#include <linux/mm.h>
  16#include <linux/interrupt.h>
  17#include <linux/slab.h>
  18#include <linux/random.h>
  19#include <linux/init.h>
  20#include <linux/delay.h>
  21#include <linux/proc_fs.h>
  22#include <linux/seq_file.h>
  23#include <linux/ftrace.h>
  24#include <linux/irq.h>
  25#include <linux/string_choices.h>
  26
  27#include <asm/ptrace.h>
  28#include <asm/processor.h>
  29#include <linux/atomic.h>
  30#include <asm/irq.h>
  31#include <asm/io.h>
  32#include <asm/iommu.h>
  33#include <asm/upa.h>
  34#include <asm/oplib.h>
  35#include <asm/prom.h>
  36#include <asm/timer.h>
  37#include <asm/smp.h>
  38#include <asm/starfire.h>
  39#include <linux/uaccess.h>
  40#include <asm/cache.h>
  41#include <asm/cpudata.h>
  42#include <asm/auxio.h>
  43#include <asm/head.h>
  44#include <asm/hypervisor.h>
  45#include <asm/cacheflush.h>
  46#include <asm/softirq_stack.h>
  47
  48#include "entry.h"
  49#include "cpumap.h"
  50#include "kstack.h"
  51
  52struct ino_bucket *ivector_table;
  53unsigned long ivector_table_pa;
  54
  55/* On several sun4u processors, it is illegal to mix bypass and
  56 * non-bypass accesses.  Therefore we access all INO buckets
  57 * using bypass accesses only.
  58 */
  59static unsigned long bucket_get_chain_pa(unsigned long bucket_pa)
  60{
  61	unsigned long ret;
  62
  63	__asm__ __volatile__("ldxa	[%1] %2, %0"
  64			     : "=&r" (ret)
  65			     : "r" (bucket_pa +
  66				    offsetof(struct ino_bucket,
  67					     __irq_chain_pa)),
  68			       "i" (ASI_PHYS_USE_EC));
  69
  70	return ret;
  71}
  72
  73static void bucket_clear_chain_pa(unsigned long bucket_pa)
  74{
  75	__asm__ __volatile__("stxa	%%g0, [%0] %1"
  76			     : /* no outputs */
  77			     : "r" (bucket_pa +
  78				    offsetof(struct ino_bucket,
  79					     __irq_chain_pa)),
  80			       "i" (ASI_PHYS_USE_EC));
  81}
  82
  83static unsigned int bucket_get_irq(unsigned long bucket_pa)
  84{
  85	unsigned int ret;
  86
  87	__asm__ __volatile__("lduwa	[%1] %2, %0"
  88			     : "=&r" (ret)
  89			     : "r" (bucket_pa +
  90				    offsetof(struct ino_bucket,
  91					     __irq)),
  92			       "i" (ASI_PHYS_USE_EC));
  93
  94	return ret;
  95}
  96
  97static void bucket_set_irq(unsigned long bucket_pa, unsigned int irq)
  98{
  99	__asm__ __volatile__("stwa	%0, [%1] %2"
 100			     : /* no outputs */
 101			     : "r" (irq),
 102			       "r" (bucket_pa +
 103				    offsetof(struct ino_bucket,
 104					     __irq)),
 105			       "i" (ASI_PHYS_USE_EC));
 106}
 107
 108#define irq_work_pa(__cpu)	&(trap_block[(__cpu)].irq_worklist_pa)
 109
 110static unsigned long hvirq_major __initdata;
 111static int __init early_hvirq_major(char *p)
 112{
 113	int rc = kstrtoul(p, 10, &hvirq_major);
 114
 115	return rc;
 116}
 117early_param("hvirq", early_hvirq_major);
 118
 119static int hv_irq_version;
 120
 121/* Major version 2.0 of HV_GRP_INTR added support for the VIRQ cookie
 122 * based interfaces, but:
 123 *
 124 * 1) Several OSs, Solaris and Linux included, use them even when only
 125 *    negotiating version 1.0 (or failing to negotiate at all).  So the
 126 *    hypervisor has a workaround that provides the VIRQ interfaces even
 127 *    when only verion 1.0 of the API is in use.
 128 *
 129 * 2) Second, and more importantly, with major version 2.0 these VIRQ
 130 *    interfaces only were actually hooked up for LDC interrupts, even
 131 *    though the Hypervisor specification clearly stated:
 132 *
 133 *	The new interrupt API functions will be available to a guest
 134 *	when it negotiates version 2.0 in the interrupt API group 0x2. When
 135 *	a guest negotiates version 2.0, all interrupt sources will only
 136 *	support using the cookie interface, and any attempt to use the
 137 *	version 1.0 interrupt APIs numbered 0xa0 to 0xa6 will result in the
 138 *	ENOTSUPPORTED error being returned.
 139 *
 140 *   with an emphasis on "all interrupt sources".
 141 *
 142 * To correct this, major version 3.0 was created which does actually
 143 * support VIRQs for all interrupt sources (not just LDC devices).  So
 144 * if we want to move completely over the cookie based VIRQs we must
 145 * negotiate major version 3.0 or later of HV_GRP_INTR.
 146 */
 147static bool sun4v_cookie_only_virqs(void)
 148{
 149	return hv_irq_version >= 3;
 150}
 151
 152static void __init irq_init_hv(void)
 153{
 154	unsigned long hv_error, major, minor = 0;
 155
 156	if (tlb_type != hypervisor)
 157		return;
 158
 159	if (hvirq_major)
 160		major = hvirq_major;
 161	else
 162		major = 3;
 163
 164	hv_error = sun4v_hvapi_register(HV_GRP_INTR, major, &minor);
 165	if (!hv_error)
 166		hv_irq_version = major;
 167	else
 168		hv_irq_version = 1;
 169
 170	pr_info("SUN4V: Using IRQ API major %d, cookie only virqs %s\n",
 171		hv_irq_version,
 172		str_enabled_disabled(sun4v_cookie_only_virqs()));
 173}
 174
 175/* This function is for the timer interrupt.*/
 176int __init arch_probe_nr_irqs(void)
 177{
 178	return 1;
 179}
 180
 181#define DEFAULT_NUM_IVECS	(0xfffU)
 182static unsigned int nr_ivec = DEFAULT_NUM_IVECS;
 183#define NUM_IVECS (nr_ivec)
 184
 185static unsigned int __init size_nr_ivec(void)
 186{
 187	if (tlb_type == hypervisor) {
 188		switch (sun4v_chip_type) {
 189		/* Athena's devhandle|devino is large.*/
 190		case SUN4V_CHIP_SPARC64X:
 191			nr_ivec = 0xffff;
 192			break;
 193		}
 194	}
 195	return nr_ivec;
 196}
 197
 198struct irq_handler_data {
 199	union {
 200		struct {
 201			unsigned int dev_handle;
 202			unsigned int dev_ino;
 203		};
 204		unsigned long sysino;
 205	};
 206	struct ino_bucket bucket;
 207	unsigned long	iclr;
 208	unsigned long	imap;
 209};
 210
 211static inline unsigned int irq_data_to_handle(struct irq_data *data)
 212{
 213	struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data);
 214
 215	return ihd->dev_handle;
 216}
 217
 218static inline unsigned int irq_data_to_ino(struct irq_data *data)
 219{
 220	struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data);
 221
 222	return ihd->dev_ino;
 223}
 224
 225static inline unsigned long irq_data_to_sysino(struct irq_data *data)
 226{
 227	struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data);
 228
 229	return ihd->sysino;
 230}
 231
 232void irq_free(unsigned int irq)
 233{
 234	void *data = irq_get_handler_data(irq);
 235
 236	kfree(data);
 237	irq_set_handler_data(irq, NULL);
 238	irq_free_descs(irq, 1);
 239}
 240
 241unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino)
 242{
 243	int irq;
 244
 245	irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL, NULL);
 246	if (irq <= 0)
 247		goto out;
 248
 249	return irq;
 250out:
 251	return 0;
 252}
 253
 254static unsigned int cookie_exists(u32 devhandle, unsigned int devino)
 255{
 256	unsigned long hv_err, cookie;
 257	struct ino_bucket *bucket;
 258	unsigned int irq = 0U;
 259
 260	hv_err = sun4v_vintr_get_cookie(devhandle, devino, &cookie);
 261	if (hv_err) {
 262		pr_err("HV get cookie failed hv_err = %ld\n", hv_err);
 263		goto out;
 264	}
 265
 266	if (cookie & ((1UL << 63UL))) {
 267		cookie = ~cookie;
 268		bucket = (struct ino_bucket *) __va(cookie);
 269		irq = bucket->__irq;
 270	}
 271out:
 272	return irq;
 273}
 274
 275static unsigned int sysino_exists(u32 devhandle, unsigned int devino)
 276{
 277	unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino);
 278	struct ino_bucket *bucket;
 279	unsigned int irq;
 280
 281	bucket = &ivector_table[sysino];
 282	irq = bucket_get_irq(__pa(bucket));
 283
 284	return irq;
 285}
 286
 287void ack_bad_irq(unsigned int irq)
 288{
 289	pr_crit("BAD IRQ ack %d\n", irq);
 290}
 291
 292void irq_install_pre_handler(int irq,
 293			     void (*func)(unsigned int, void *, void *),
 294			     void *arg1, void *arg2)
 295{
 296	pr_warn("IRQ pre handler NOT supported.\n");
 297}
 298
 299/*
 300 * /proc/interrupts printing:
 301 */
 302int arch_show_interrupts(struct seq_file *p, int prec)
 303{
 304	int j;
 305
 306	seq_printf(p, "NMI:");
 307	for_each_online_cpu(j)
 308		seq_put_decimal_ull_width(p, " ", cpu_data(j).__nmi_count, 10);
 309	seq_printf(p, "     Non-maskable interrupts\n");
 310	return 0;
 311}
 312
 313static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid)
 314{
 315	unsigned int tid;
 316
 317	if (this_is_starfire) {
 318		tid = starfire_translate(imap, cpuid);
 319		tid <<= IMAP_TID_SHIFT;
 320		tid &= IMAP_TID_UPA;
 321	} else {
 322		if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 323			unsigned long ver;
 324
 325			__asm__ ("rdpr %%ver, %0" : "=r" (ver));
 326			if ((ver >> 32UL) == __JALAPENO_ID ||
 327			    (ver >> 32UL) == __SERRANO_ID) {
 328				tid = cpuid << IMAP_TID_SHIFT;
 329				tid &= IMAP_TID_JBUS;
 330			} else {
 331				unsigned int a = cpuid & 0x1f;
 332				unsigned int n = (cpuid >> 5) & 0x1f;
 333
 334				tid = ((a << IMAP_AID_SHIFT) |
 335				       (n << IMAP_NID_SHIFT));
 336				tid &= (IMAP_AID_SAFARI |
 337					IMAP_NID_SAFARI);
 338			}
 339		} else {
 340			tid = cpuid << IMAP_TID_SHIFT;
 341			tid &= IMAP_TID_UPA;
 342		}
 343	}
 344
 345	return tid;
 346}
 347
 348#ifdef CONFIG_SMP
 349static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity)
 350{
 351	int cpuid;
 352
 353	if (cpumask_equal(affinity, cpu_online_mask)) {
 354		cpuid = map_to_cpu(irq);
 355	} else {
 356		cpuid = cpumask_first_and(affinity, cpu_online_mask);
 357		cpuid = cpuid < nr_cpu_ids ? cpuid : map_to_cpu(irq);
 358	}
 359
 360	return cpuid;
 361}
 362#else
 363#define irq_choose_cpu(irq, affinity)	\
 364	real_hard_smp_processor_id()
 365#endif
 366
 367static void sun4u_irq_enable(struct irq_data *data)
 368{
 369	struct irq_handler_data *handler_data;
 370
 371	handler_data = irq_data_get_irq_handler_data(data);
 372	if (likely(handler_data)) {
 373		unsigned long cpuid, imap, val;
 374		unsigned int tid;
 375
 376		cpuid = irq_choose_cpu(data->irq,
 377				       irq_data_get_affinity_mask(data));
 378		imap = handler_data->imap;
 379
 380		tid = sun4u_compute_tid(imap, cpuid);
 381
 382		val = upa_readq(imap);
 383		val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS |
 384			 IMAP_AID_SAFARI | IMAP_NID_SAFARI);
 385		val |= tid | IMAP_VALID;
 386		upa_writeq(val, imap);
 387		upa_writeq(ICLR_IDLE, handler_data->iclr);
 388	}
 389}
 390
 391static int sun4u_set_affinity(struct irq_data *data,
 392			       const struct cpumask *mask, bool force)
 393{
 394	struct irq_handler_data *handler_data;
 395
 396	handler_data = irq_data_get_irq_handler_data(data);
 397	if (likely(handler_data)) {
 398		unsigned long cpuid, imap, val;
 399		unsigned int tid;
 400
 401		cpuid = irq_choose_cpu(data->irq, mask);
 402		imap = handler_data->imap;
 403
 404		tid = sun4u_compute_tid(imap, cpuid);
 405
 406		val = upa_readq(imap);
 407		val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS |
 408			 IMAP_AID_SAFARI | IMAP_NID_SAFARI);
 409		val |= tid | IMAP_VALID;
 410		upa_writeq(val, imap);
 411		upa_writeq(ICLR_IDLE, handler_data->iclr);
 412	}
 413
 414	return 0;
 415}
 416
 417/* Don't do anything.  The desc->status check for IRQ_DISABLED in
 418 * handler_irq() will skip the handler call and that will leave the
 419 * interrupt in the sent state.  The next ->enable() call will hit the
 420 * ICLR register to reset the state machine.
 421 *
 422 * This scheme is necessary, instead of clearing the Valid bit in the
 423 * IMAP register, to handle the case of IMAP registers being shared by
 424 * multiple INOs (and thus ICLR registers).  Since we use a different
 425 * virtual IRQ for each shared IMAP instance, the generic code thinks
 426 * there is only one user so it prematurely calls ->disable() on
 427 * free_irq().
 428 *
 429 * We have to provide an explicit ->disable() method instead of using
 430 * NULL to get the default.  The reason is that if the generic code
 431 * sees that, it also hooks up a default ->shutdown method which
 432 * invokes ->mask() which we do not want.  See irq_chip_set_defaults().
 433 */
 434static void sun4u_irq_disable(struct irq_data *data)
 435{
 436}
 437
 438static void sun4u_irq_eoi(struct irq_data *data)
 439{
 440	struct irq_handler_data *handler_data;
 441
 442	handler_data = irq_data_get_irq_handler_data(data);
 443	if (likely(handler_data))
 444		upa_writeq(ICLR_IDLE, handler_data->iclr);
 445}
 446
 447static void sun4v_irq_enable(struct irq_data *data)
 448{
 449	unsigned long cpuid = irq_choose_cpu(data->irq,
 450					     irq_data_get_affinity_mask(data));
 451	unsigned int ino = irq_data_to_sysino(data);
 452	int err;
 453
 454	err = sun4v_intr_settarget(ino, cpuid);
 455	if (err != HV_EOK)
 456		printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): "
 457		       "err(%d)\n", ino, cpuid, err);
 458	err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
 459	if (err != HV_EOK)
 460		printk(KERN_ERR "sun4v_intr_setstate(%x): "
 461		       "err(%d)\n", ino, err);
 462	err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED);
 463	if (err != HV_EOK)
 464		printk(KERN_ERR "sun4v_intr_setenabled(%x): err(%d)\n",
 465		       ino, err);
 466}
 467
 468static int sun4v_set_affinity(struct irq_data *data,
 469			       const struct cpumask *mask, bool force)
 470{
 471	unsigned long cpuid = irq_choose_cpu(data->irq, mask);
 472	unsigned int ino = irq_data_to_sysino(data);
 473	int err;
 474
 475	err = sun4v_intr_settarget(ino, cpuid);
 476	if (err != HV_EOK)
 477		printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): "
 478		       "err(%d)\n", ino, cpuid, err);
 479
 480	return 0;
 481}
 482
 483static void sun4v_irq_disable(struct irq_data *data)
 484{
 485	unsigned int ino = irq_data_to_sysino(data);
 486	int err;
 487
 488	err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED);
 489	if (err != HV_EOK)
 490		printk(KERN_ERR "sun4v_intr_setenabled(%x): "
 491		       "err(%d)\n", ino, err);
 492}
 493
 494static void sun4v_irq_eoi(struct irq_data *data)
 495{
 496	unsigned int ino = irq_data_to_sysino(data);
 497	int err;
 498
 499	err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
 500	if (err != HV_EOK)
 501		printk(KERN_ERR "sun4v_intr_setstate(%x): "
 502		       "err(%d)\n", ino, err);
 503}
 504
 505static void sun4v_virq_enable(struct irq_data *data)
 506{
 507	unsigned long dev_handle = irq_data_to_handle(data);
 508	unsigned long dev_ino = irq_data_to_ino(data);
 509	unsigned long cpuid;
 510	int err;
 511
 512	cpuid = irq_choose_cpu(data->irq, irq_data_get_affinity_mask(data));
 513
 514	err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
 515	if (err != HV_EOK)
 516		printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
 517		       "err(%d)\n",
 518		       dev_handle, dev_ino, cpuid, err);
 519	err = sun4v_vintr_set_state(dev_handle, dev_ino,
 520				    HV_INTR_STATE_IDLE);
 521	if (err != HV_EOK)
 522		printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
 523		       "HV_INTR_STATE_IDLE): err(%d)\n",
 524		       dev_handle, dev_ino, err);
 525	err = sun4v_vintr_set_valid(dev_handle, dev_ino,
 526				    HV_INTR_ENABLED);
 527	if (err != HV_EOK)
 528		printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
 529		       "HV_INTR_ENABLED): err(%d)\n",
 530		       dev_handle, dev_ino, err);
 531}
 532
 533static int sun4v_virt_set_affinity(struct irq_data *data,
 534				    const struct cpumask *mask, bool force)
 535{
 536	unsigned long dev_handle = irq_data_to_handle(data);
 537	unsigned long dev_ino = irq_data_to_ino(data);
 538	unsigned long cpuid;
 539	int err;
 540
 541	cpuid = irq_choose_cpu(data->irq, mask);
 542
 543	err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
 544	if (err != HV_EOK)
 545		printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
 546		       "err(%d)\n",
 547		       dev_handle, dev_ino, cpuid, err);
 548
 549	return 0;
 550}
 551
 552static void sun4v_virq_disable(struct irq_data *data)
 553{
 554	unsigned long dev_handle = irq_data_to_handle(data);
 555	unsigned long dev_ino = irq_data_to_ino(data);
 556	int err;
 557
 558
 559	err = sun4v_vintr_set_valid(dev_handle, dev_ino,
 560				    HV_INTR_DISABLED);
 561	if (err != HV_EOK)
 562		printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
 563		       "HV_INTR_DISABLED): err(%d)\n",
 564		       dev_handle, dev_ino, err);
 565}
 566
 567static void sun4v_virq_eoi(struct irq_data *data)
 568{
 569	unsigned long dev_handle = irq_data_to_handle(data);
 570	unsigned long dev_ino = irq_data_to_ino(data);
 571	int err;
 572
 573	err = sun4v_vintr_set_state(dev_handle, dev_ino,
 574				    HV_INTR_STATE_IDLE);
 575	if (err != HV_EOK)
 576		printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
 577		       "HV_INTR_STATE_IDLE): err(%d)\n",
 578		       dev_handle, dev_ino, err);
 579}
 580
 581static struct irq_chip sun4u_irq = {
 582	.name			= "sun4u",
 583	.irq_enable		= sun4u_irq_enable,
 584	.irq_disable		= sun4u_irq_disable,
 585	.irq_eoi		= sun4u_irq_eoi,
 586	.irq_set_affinity	= sun4u_set_affinity,
 587	.flags			= IRQCHIP_EOI_IF_HANDLED,
 588};
 589
 590static struct irq_chip sun4v_irq = {
 591	.name			= "sun4v",
 592	.irq_enable		= sun4v_irq_enable,
 593	.irq_disable		= sun4v_irq_disable,
 594	.irq_eoi		= sun4v_irq_eoi,
 595	.irq_set_affinity	= sun4v_set_affinity,
 596	.flags			= IRQCHIP_EOI_IF_HANDLED,
 597};
 598
 599static struct irq_chip sun4v_virq = {
 600	.name			= "vsun4v",
 601	.irq_enable		= sun4v_virq_enable,
 602	.irq_disable		= sun4v_virq_disable,
 603	.irq_eoi		= sun4v_virq_eoi,
 604	.irq_set_affinity	= sun4v_virt_set_affinity,
 605	.flags			= IRQCHIP_EOI_IF_HANDLED,
 606};
 607
 608unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap)
 609{
 610	struct irq_handler_data *handler_data;
 611	struct ino_bucket *bucket;
 612	unsigned int irq;
 613	int ino;
 614
 615	BUG_ON(tlb_type == hypervisor);
 616
 617	ino = (upa_readq(imap) & (IMAP_IGN | IMAP_INO)) + inofixup;
 618	bucket = &ivector_table[ino];
 619	irq = bucket_get_irq(__pa(bucket));
 620	if (!irq) {
 621		irq = irq_alloc(0, ino);
 622		bucket_set_irq(__pa(bucket), irq);
 623		irq_set_chip_and_handler_name(irq, &sun4u_irq,
 624					      handle_fasteoi_irq, "IVEC");
 625	}
 626
 627	handler_data = irq_get_handler_data(irq);
 628	if (unlikely(handler_data))
 629		goto out;
 630
 631	handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
 632	if (unlikely(!handler_data)) {
 633		prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n");
 634		prom_halt();
 635	}
 636	irq_set_handler_data(irq, handler_data);
 637
 638	handler_data->imap  = imap;
 639	handler_data->iclr  = iclr;
 640
 641out:
 642	return irq;
 643}
 644
 645static unsigned int sun4v_build_common(u32 devhandle, unsigned int devino,
 646		void (*handler_data_init)(struct irq_handler_data *data,
 647		u32 devhandle, unsigned int devino),
 648		struct irq_chip *chip)
 649{
 650	struct irq_handler_data *data;
 651	unsigned int irq;
 652
 653	irq = irq_alloc(devhandle, devino);
 654	if (!irq)
 655		goto out;
 656
 657	data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
 658	if (unlikely(!data)) {
 659		pr_err("IRQ handler data allocation failed.\n");
 660		irq_free(irq);
 661		irq = 0;
 662		goto out;
 663	}
 664
 665	irq_set_handler_data(irq, data);
 666	handler_data_init(data, devhandle, devino);
 667	irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, "IVEC");
 668	data->imap = ~0UL;
 669	data->iclr = ~0UL;
 670out:
 671	return irq;
 672}
 673
 674static unsigned long cookie_assign(unsigned int irq, u32 devhandle,
 675		unsigned int devino)
 676{
 677	struct irq_handler_data *ihd = irq_get_handler_data(irq);
 678	unsigned long hv_error, cookie;
 679
 680	/* handler_irq needs to find the irq. cookie is seen signed in
 681	 * sun4v_dev_mondo and treated as a non ivector_table delivery.
 682	 */
 683	ihd->bucket.__irq = irq;
 684	cookie = ~__pa(&ihd->bucket);
 685
 686	hv_error = sun4v_vintr_set_cookie(devhandle, devino, cookie);
 687	if (hv_error)
 688		pr_err("HV vintr set cookie failed = %ld\n", hv_error);
 689
 690	return hv_error;
 691}
 692
 693static void cookie_handler_data(struct irq_handler_data *data,
 694				u32 devhandle, unsigned int devino)
 695{
 696	data->dev_handle = devhandle;
 697	data->dev_ino = devino;
 698}
 699
 700static unsigned int cookie_build_irq(u32 devhandle, unsigned int devino,
 701				     struct irq_chip *chip)
 702{
 703	unsigned long hv_error;
 704	unsigned int irq;
 705
 706	irq = sun4v_build_common(devhandle, devino, cookie_handler_data, chip);
 707
 708	hv_error = cookie_assign(irq, devhandle, devino);
 709	if (hv_error) {
 710		irq_free(irq);
 711		irq = 0;
 712	}
 713
 714	return irq;
 715}
 716
 717static unsigned int sun4v_build_cookie(u32 devhandle, unsigned int devino)
 718{
 719	unsigned int irq;
 720
 721	irq = cookie_exists(devhandle, devino);
 722	if (irq)
 723		goto out;
 724
 725	irq = cookie_build_irq(devhandle, devino, &sun4v_virq);
 726
 727out:
 728	return irq;
 729}
 730
 731static void sysino_set_bucket(unsigned int irq)
 732{
 733	struct irq_handler_data *ihd = irq_get_handler_data(irq);
 734	struct ino_bucket *bucket;
 735	unsigned long sysino;
 736
 737	sysino = sun4v_devino_to_sysino(ihd->dev_handle, ihd->dev_ino);
 738	BUG_ON(sysino >= nr_ivec);
 739	bucket = &ivector_table[sysino];
 740	bucket_set_irq(__pa(bucket), irq);
 741}
 742
 743static void sysino_handler_data(struct irq_handler_data *data,
 744				u32 devhandle, unsigned int devino)
 745{
 746	unsigned long sysino;
 747
 748	sysino = sun4v_devino_to_sysino(devhandle, devino);
 749	data->sysino = sysino;
 750}
 751
 752static unsigned int sysino_build_irq(u32 devhandle, unsigned int devino,
 753				     struct irq_chip *chip)
 754{
 755	unsigned int irq;
 756
 757	irq = sun4v_build_common(devhandle, devino, sysino_handler_data, chip);
 758	if (!irq)
 759		goto out;
 760
 761	sysino_set_bucket(irq);
 762out:
 763	return irq;
 764}
 765
 766static int sun4v_build_sysino(u32 devhandle, unsigned int devino)
 767{
 768	int irq;
 769
 770	irq = sysino_exists(devhandle, devino);
 771	if (irq)
 772		goto out;
 773
 774	irq = sysino_build_irq(devhandle, devino, &sun4v_irq);
 775out:
 776	return irq;
 777}
 778
 779unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino)
 780{
 781	unsigned int irq;
 782
 783	if (sun4v_cookie_only_virqs())
 784		irq = sun4v_build_cookie(devhandle, devino);
 785	else
 786		irq = sun4v_build_sysino(devhandle, devino);
 787
 788	return irq;
 789}
 790
 791unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino)
 792{
 793	int irq;
 794
 795	irq = cookie_build_irq(devhandle, devino, &sun4v_virq);
 796	if (!irq)
 797		goto out;
 798
 799	/* This is borrowed from the original function.
 800	 */
 801	irq_set_status_flags(irq, IRQ_NOAUTOEN);
 802
 803out:
 804	return irq;
 805}
 806
 807void *hardirq_stack[NR_CPUS];
 808void *softirq_stack[NR_CPUS];
 809
 810void __irq_entry handler_irq(int pil, struct pt_regs *regs)
 811{
 812	unsigned long pstate, bucket_pa;
 813	struct pt_regs *old_regs;
 814	void *orig_sp;
 815
 816	clear_softint(1 << pil);
 817
 818	old_regs = set_irq_regs(regs);
 819	irq_enter();
 820
 821	/* Grab an atomic snapshot of the pending IVECs.  */
 822	__asm__ __volatile__("rdpr	%%pstate, %0\n\t"
 823			     "wrpr	%0, %3, %%pstate\n\t"
 824			     "ldx	[%2], %1\n\t"
 825			     "stx	%%g0, [%2]\n\t"
 826			     "wrpr	%0, 0x0, %%pstate\n\t"
 827			     : "=&r" (pstate), "=&r" (bucket_pa)
 828			     : "r" (irq_work_pa(smp_processor_id())),
 829			       "i" (PSTATE_IE)
 830			     : "memory");
 831
 832	orig_sp = set_hardirq_stack();
 833
 834	while (bucket_pa) {
 835		unsigned long next_pa;
 836		unsigned int irq;
 837
 838		next_pa = bucket_get_chain_pa(bucket_pa);
 839		irq = bucket_get_irq(bucket_pa);
 840		bucket_clear_chain_pa(bucket_pa);
 841
 842		generic_handle_irq(irq);
 843
 844		bucket_pa = next_pa;
 845	}
 846
 847	restore_hardirq_stack(orig_sp);
 848
 849	irq_exit();
 850	set_irq_regs(old_regs);
 851}
 852
 853#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK
 854void do_softirq_own_stack(void)
 855{
 856	void *orig_sp, *sp = softirq_stack[smp_processor_id()];
 857
 858	sp += THREAD_SIZE - 192 - STACK_BIAS;
 859
 860	__asm__ __volatile__("mov %%sp, %0\n\t"
 861			     "mov %1, %%sp"
 862			     : "=&r" (orig_sp)
 863			     : "r" (sp));
 864	__do_softirq();
 865	__asm__ __volatile__("mov %0, %%sp"
 866			     : : "r" (orig_sp));
 867}
 868#endif
 869
 870#ifdef CONFIG_HOTPLUG_CPU
 871void fixup_irqs(void)
 872{
 873	unsigned int irq;
 874
 875	for (irq = 0; irq < NR_IRQS; irq++) {
 876		struct irq_desc *desc = irq_to_desc(irq);
 877		struct irq_data *data;
 878		unsigned long flags;
 879
 880		if (!desc)
 881			continue;
 882		data = irq_desc_get_irq_data(desc);
 883		raw_spin_lock_irqsave(&desc->lock, flags);
 884		if (desc->action && !irqd_is_per_cpu(data)) {
 885			if (data->chip->irq_set_affinity)
 886				data->chip->irq_set_affinity(data,
 887					irq_data_get_affinity_mask(data),
 888					false);
 889		}
 890		raw_spin_unlock_irqrestore(&desc->lock, flags);
 891	}
 892
 893	tick_ops->disable_irq();
 894}
 895#endif
 896
 897struct sun5_timer {
 898	u64	count0;
 899	u64	limit0;
 900	u64	count1;
 901	u64	limit1;
 902};
 903
 904static struct sun5_timer *prom_timers;
 905static u64 prom_limit0, prom_limit1;
 906
 907static void map_prom_timers(void)
 908{
 909	struct device_node *dp;
 910	const unsigned int *addr;
 911
 912	/* PROM timer node hangs out in the top level of device siblings... */
 913	dp = of_find_node_by_path("/");
 914	dp = dp->child;
 915	while (dp) {
 916		if (of_node_name_eq(dp, "counter-timer"))
 917			break;
 918		dp = dp->sibling;
 919	}
 920
 921	/* Assume if node is not present, PROM uses different tick mechanism
 922	 * which we should not care about.
 923	 */
 924	if (!dp) {
 925		prom_timers = (struct sun5_timer *) 0;
 926		return;
 927	}
 928
 929	/* If PROM is really using this, it must be mapped by him. */
 930	addr = of_get_property(dp, "address", NULL);
 931	if (!addr) {
 932		prom_printf("PROM does not have timer mapped, trying to continue.\n");
 933		prom_timers = (struct sun5_timer *) 0;
 934		return;
 935	}
 936	prom_timers = (struct sun5_timer *) ((unsigned long)addr[0]);
 937}
 938
 939static void kill_prom_timer(void)
 940{
 941	if (!prom_timers)
 942		return;
 943
 944	/* Save them away for later. */
 945	prom_limit0 = prom_timers->limit0;
 946	prom_limit1 = prom_timers->limit1;
 947
 948	/* Just as in sun4c PROM uses timer which ticks at IRQ 14.
 949	 * We turn both off here just to be paranoid.
 950	 */
 951	prom_timers->limit0 = 0;
 952	prom_timers->limit1 = 0;
 953
 954	/* Wheee, eat the interrupt packet too... */
 955	__asm__ __volatile__(
 956"	mov	0x40, %%g2\n"
 957"	ldxa	[%%g0] %0, %%g1\n"
 958"	ldxa	[%%g2] %1, %%g1\n"
 959"	stxa	%%g0, [%%g0] %0\n"
 960"	membar	#Sync\n"
 961	: /* no outputs */
 962	: "i" (ASI_INTR_RECEIVE), "i" (ASI_INTR_R)
 963	: "g1", "g2");
 964}
 965
 966void notrace init_irqwork_curcpu(void)
 967{
 968	int cpu = hard_smp_processor_id();
 969
 970	trap_block[cpu].irq_worklist_pa = 0UL;
 971}
 972
 973/* Please be very careful with register_one_mondo() and
 974 * sun4v_register_mondo_queues().
 975 *
 976 * On SMP this gets invoked from the CPU trampoline before
 977 * the cpu has fully taken over the trap table from OBP,
 978 * and its kernel stack + %g6 thread register state is
 979 * not fully cooked yet.
 980 *
 981 * Therefore you cannot make any OBP calls, not even prom_printf,
 982 * from these two routines.
 983 */
 984static void notrace register_one_mondo(unsigned long paddr, unsigned long type,
 985				       unsigned long qmask)
 986{
 987	unsigned long num_entries = (qmask + 1) / 64;
 988	unsigned long status;
 989
 990	status = sun4v_cpu_qconf(type, paddr, num_entries);
 991	if (status != HV_EOK) {
 992		prom_printf("SUN4V: sun4v_cpu_qconf(%lu:%lx:%lu) failed, "
 993			    "err %lu\n", type, paddr, num_entries, status);
 994		prom_halt();
 995	}
 996}
 997
 998void notrace sun4v_register_mondo_queues(int this_cpu)
 999{
1000	struct trap_per_cpu *tb = &trap_block[this_cpu];
1001
1002	register_one_mondo(tb->cpu_mondo_pa, HV_CPU_QUEUE_CPU_MONDO,
1003			   tb->cpu_mondo_qmask);
1004	register_one_mondo(tb->dev_mondo_pa, HV_CPU_QUEUE_DEVICE_MONDO,
1005			   tb->dev_mondo_qmask);
1006	register_one_mondo(tb->resum_mondo_pa, HV_CPU_QUEUE_RES_ERROR,
1007			   tb->resum_qmask);
1008	register_one_mondo(tb->nonresum_mondo_pa, HV_CPU_QUEUE_NONRES_ERROR,
1009			   tb->nonresum_qmask);
1010}
1011
1012/* Each queue region must be a power of 2 multiple of 64 bytes in
1013 * size.  The base real address must be aligned to the size of the
1014 * region.  Thus, an 8KB queue must be 8KB aligned, for example.
1015 */
1016static void __init alloc_one_queue(unsigned long *pa_ptr, unsigned long qmask)
1017{
1018	unsigned long size = PAGE_ALIGN(qmask + 1);
1019	unsigned long order = get_order(size);
1020	unsigned long p;
1021
1022	p = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
1023	if (!p) {
1024		prom_printf("SUN4V: Error, cannot allocate queue.\n");
1025		prom_halt();
1026	}
1027
1028	*pa_ptr = __pa(p);
1029}
1030
1031static void __init init_cpu_send_mondo_info(struct trap_per_cpu *tb)
1032{
1033#ifdef CONFIG_SMP
1034	unsigned long page;
1035	void *mondo, *p;
1036
1037	BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > PAGE_SIZE);
1038
1039	/* Make sure mondo block is 64byte aligned */
1040	p = kzalloc(127, GFP_KERNEL);
1041	if (!p) {
1042		prom_printf("SUN4V: Error, cannot allocate mondo block.\n");
1043		prom_halt();
1044	}
1045	mondo = (void *)(((unsigned long)p + 63) & ~0x3f);
1046	tb->cpu_mondo_block_pa = __pa(mondo);
1047
1048	page = get_zeroed_page(GFP_KERNEL);
1049	if (!page) {
1050		prom_printf("SUN4V: Error, cannot allocate cpu list page.\n");
1051		prom_halt();
1052	}
1053
1054	tb->cpu_list_pa = __pa(page);
1055#endif
1056}
1057
1058/* Allocate mondo and error queues for all possible cpus.  */
1059static void __init sun4v_init_mondo_queues(void)
1060{
1061	int cpu;
1062
1063	for_each_possible_cpu(cpu) {
1064		struct trap_per_cpu *tb = &trap_block[cpu];
1065
1066		alloc_one_queue(&tb->cpu_mondo_pa, tb->cpu_mondo_qmask);
1067		alloc_one_queue(&tb->dev_mondo_pa, tb->dev_mondo_qmask);
1068		alloc_one_queue(&tb->resum_mondo_pa, tb->resum_qmask);
1069		alloc_one_queue(&tb->resum_kernel_buf_pa, tb->resum_qmask);
1070		alloc_one_queue(&tb->nonresum_mondo_pa, tb->nonresum_qmask);
1071		alloc_one_queue(&tb->nonresum_kernel_buf_pa,
1072				tb->nonresum_qmask);
1073	}
1074}
1075
1076static void __init init_send_mondo_info(void)
1077{
1078	int cpu;
1079
1080	for_each_possible_cpu(cpu) {
1081		struct trap_per_cpu *tb = &trap_block[cpu];
1082
1083		init_cpu_send_mondo_info(tb);
1084	}
1085}
1086
1087static struct irqaction timer_irq_action = {
1088	.name = "timer",
1089};
1090
1091static void __init irq_ivector_init(void)
1092{
1093	unsigned long size, order;
1094	unsigned int ivecs;
1095
1096	/* If we are doing cookie only VIRQs then we do not need the ivector
1097	 * table to process interrupts.
1098	 */
1099	if (sun4v_cookie_only_virqs())
1100		return;
1101
1102	ivecs = size_nr_ivec();
1103	size = sizeof(struct ino_bucket) * ivecs;
1104	order = get_order(size);
1105	ivector_table = (struct ino_bucket *)
1106		__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
1107	if (!ivector_table) {
1108		prom_printf("Fatal error, cannot allocate ivector_table\n");
1109		prom_halt();
1110	}
1111	__flush_dcache_range((unsigned long) ivector_table,
1112			     ((unsigned long) ivector_table) + size);
1113
1114	ivector_table_pa = __pa(ivector_table);
1115}
1116
1117/* Only invoked on boot processor.*/
1118void __init init_IRQ(void)
1119{
1120	irq_init_hv();
1121	irq_ivector_init();
1122	map_prom_timers();
1123	kill_prom_timer();
1124
1125	if (tlb_type == hypervisor)
1126		sun4v_init_mondo_queues();
1127
1128	init_send_mondo_info();
1129
1130	if (tlb_type == hypervisor) {
1131		/* Load up the boot cpu's entries.  */
1132		sun4v_register_mondo_queues(hard_smp_processor_id());
1133	}
1134
1135	/* We need to clear any IRQ's pending in the soft interrupt
1136	 * registers, a spurious one could be left around from the
1137	 * PROM timer which we just disabled.
1138	 */
1139	clear_softint(get_softint());
1140
1141	/* Now that ivector table is initialized, it is safe
1142	 * to receive IRQ vector traps.  We will normally take
1143	 * one or two right now, in case some device PROM used
1144	 * to boot us wants to speak to us.  We just ignore them.
1145	 */
1146	__asm__ __volatile__("rdpr	%%pstate, %%g1\n\t"
1147			     "or	%%g1, %0, %%g1\n\t"
1148			     "wrpr	%%g1, 0x0, %%pstate"
1149			     : /* No outputs */
1150			     : "i" (PSTATE_IE)
1151			     : "g1");
1152
1153	irq_to_desc(0)->action = &timer_irq_action;
1154}