FILE
events_base.c
File preview with syntax highlighting.
linux-6.19.14/drivers/xen/events/events_base.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Xen event channels 4 * 5 * Xen models interrupts with abstract event channels. Because each 6 * domain gets 1024 event channels, but NR_IRQ is not that large, we 7 * must dynamically map irqs<->event channels. The event channels 8 * interface with the rest of the kernel by defining a xen interrupt 9 * chip. When an event is received, it is mapped to an irq and sent 10 * through the normal interrupt processing path. 11 * 12 * There are four kinds of events which can be mapped to an event 13 * channel: 14 * 15 * 1. Inter-domain notifications. This includes all the virtual 16 * device events, since they're driven by front-ends in another domain 17 * (typically dom0). 18 * 2. VIRQs, typically used for timers. These are per-cpu events. 19 * 3. IPIs. 20 * 4. PIRQs - Hardware interrupts. 21 * 22 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 23 */ 24 25#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 26 27#include <linux/linkage.h> 28#include <linux/interrupt.h> 29#include <linux/irq.h> 30#include <linux/moduleparam.h> 31#include <linux/string.h> 32#include <linux/memblock.h> 33#include <linux/slab.h> 34#include <linux/irqnr.h> 35#include <linux/pci.h> 36#include <linux/rcupdate.h> 37#include <linux/spinlock.h> 38#include <linux/cpuhotplug.h> 39#include <linux/atomic.h> 40#include <linux/ktime.h> 41 42#ifdef CONFIG_X86 43#include <asm/desc.h> 44#include <asm/ptrace.h> 45#include <asm/idtentry.h> 46#include <asm/irq.h> 47#include <asm/io_apic.h> 48#include <asm/i8259.h> 49#include <asm/xen/cpuid.h> 50#include <asm/xen/pci.h> 51#endif 52#include <asm/sync_bitops.h> 53#include <asm/xen/hypercall.h> 54#include <asm/xen/hypervisor.h> 55#include <xen/page.h> 56 57#include <xen/xen.h> 58#include <xen/hvm.h> 59#include <xen/xen-ops.h> 60#include <xen/events.h> 61#include <xen/interface/xen.h> 62#include <xen/interface/event_channel.h> 63#include <xen/interface/hvm/hvm_op.h> 64#include <xen/interface/hvm/params.h> 65#include <xen/interface/physdev.h> 66#include <xen/interface/sched.h> 67#include <xen/interface/vcpu.h> 68#include <xen/xenbus.h> 69#include <asm/hw_irq.h> 70 71#include "events_internal.h" 72 73#undef MODULE_PARAM_PREFIX 74#define MODULE_PARAM_PREFIX "xen." 75 76/* Interrupt types. */ 77enum xen_irq_type { 78 IRQT_UNBOUND = 0, 79 IRQT_PIRQ, 80 IRQT_VIRQ, 81 IRQT_IPI, 82 IRQT_EVTCHN 83}; 84 85/* 86 * Packed IRQ information: 87 * type - enum xen_irq_type 88 * event channel - irq->event channel mapping 89 * cpu - cpu this event channel is bound to 90 * index - type-specific information: 91 * PIRQ - vector, with MSB being "needs EIO", or physical IRQ of the HVM 92 * guest, or GSI (real passthrough IRQ) of the device. 93 * VIRQ - virq number 94 * IPI - IPI vector 95 * EVTCHN - 96 */ 97struct irq_info { 98 struct list_head list; 99 struct list_head eoi_list; 100 struct rcu_work rwork; 101 short refcnt; 102 u8 spurious_cnt; 103 u8 is_accounted; 104 short type; /* type: IRQT_* */ 105 u8 mask_reason; /* Why is event channel masked */ 106#define EVT_MASK_REASON_EXPLICIT 0x01 107#define EVT_MASK_REASON_TEMPORARY 0x02 108#define EVT_MASK_REASON_EOI_PENDING 0x04 109 u8 is_active; /* Is event just being handled? */ 110 unsigned irq; 111 evtchn_port_t evtchn; /* event channel */ 112 unsigned short cpu; /* cpu bound */ 113 unsigned short eoi_cpu; /* EOI must happen on this cpu-1 */ 114 unsigned int irq_epoch; /* If eoi_cpu valid: irq_epoch of event */ 115 u64 eoi_time; /* Time in jiffies when to EOI. */ 116 raw_spinlock_t lock; 117 bool is_static; /* Is event channel static */ 118 119 union { 120 unsigned short virq; 121 enum ipi_vector ipi; 122 struct { 123 unsigned short pirq; 124 unsigned short gsi; 125 unsigned char vector; 126 unsigned char flags; 127 uint16_t domid; 128 } pirq; 129 struct xenbus_device *interdomain; 130 } u; 131}; 132 133#define PIRQ_NEEDS_EOI (1 << 0) 134#define PIRQ_SHAREABLE (1 << 1) 135#define PIRQ_MSI_GROUP (1 << 2) 136 137static uint __read_mostly event_loop_timeout = 2; 138module_param(event_loop_timeout, uint, 0644); 139 140static uint __read_mostly event_eoi_delay = 10; 141module_param(event_eoi_delay, uint, 0644); 142 143const struct evtchn_ops *evtchn_ops; 144 145/* 146 * This lock protects updates to the following mapping and reference-count 147 * arrays. The lock does not need to be acquired to read the mapping tables. 148 */ 149static DEFINE_MUTEX(irq_mapping_update_lock); 150 151/* 152 * Lock hierarchy: 153 * 154 * irq_mapping_update_lock 155 * IRQ-desc lock 156 * percpu eoi_list_lock 157 * irq_info->lock 158 */ 159 160static LIST_HEAD(xen_irq_list_head); 161 162/* IRQ <-> VIRQ mapping. */ 163static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1}; 164 165/* IRQ <-> IPI mapping */ 166static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1}; 167/* Cache for IPI event channels - needed for hot cpu unplug (avoid RCU usage). */ 168static DEFINE_PER_CPU(evtchn_port_t [XEN_NR_IPIS], ipi_to_evtchn) = {[0 ... XEN_NR_IPIS-1] = 0}; 169 170/* Event channel distribution data */ 171static atomic_t channels_on_cpu[NR_CPUS]; 172 173static int **evtchn_to_irq; 174#ifdef CONFIG_X86 175static unsigned long *pirq_eoi_map; 176#endif 177static bool (*pirq_needs_eoi)(struct irq_info *info); 178 179#define EVTCHN_ROW(e) (e / (PAGE_SIZE/sizeof(**evtchn_to_irq))) 180#define EVTCHN_COL(e) (e % (PAGE_SIZE/sizeof(**evtchn_to_irq))) 181#define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq)) 182 183/* Xen will never allocate port zero for any purpose. */ 184#define VALID_EVTCHN(chn) ((chn) != 0) 185 186static struct irq_info *legacy_info_ptrs[NR_IRQS_LEGACY]; 187 188static struct irq_chip xen_dynamic_chip; 189static struct irq_chip xen_lateeoi_chip; 190static struct irq_chip xen_percpu_chip; 191static struct irq_chip xen_pirq_chip; 192static void enable_dynirq(struct irq_data *data); 193 194static DEFINE_PER_CPU(unsigned int, irq_epoch); 195 196static void clear_evtchn_to_irq_row(int *evtchn_row) 197{ 198 unsigned col; 199 200 for (col = 0; col < EVTCHN_PER_ROW; col++) 201 WRITE_ONCE(evtchn_row[col], -1); 202} 203 204static void clear_evtchn_to_irq_all(void) 205{ 206 unsigned row; 207 208 for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) { 209 if (evtchn_to_irq[row] == NULL) 210 continue; 211 clear_evtchn_to_irq_row(evtchn_to_irq[row]); 212 } 213} 214 215static int set_evtchn_to_irq(evtchn_port_t evtchn, unsigned int irq) 216{ 217 unsigned row; 218 unsigned col; 219 int *evtchn_row; 220 221 if (evtchn >= xen_evtchn_max_channels()) 222 return -EINVAL; 223 224 row = EVTCHN_ROW(evtchn); 225 col = EVTCHN_COL(evtchn); 226 227 if (evtchn_to_irq[row] == NULL) { 228 /* Unallocated irq entries return -1 anyway */ 229 if (irq == -1) 230 return 0; 231 232 evtchn_row = (int *) __get_free_pages(GFP_KERNEL, 0); 233 if (evtchn_row == NULL) 234 return -ENOMEM; 235 236 clear_evtchn_to_irq_row(evtchn_row); 237 238 /* 239 * We've prepared an empty row for the mapping. If a different 240 * thread was faster inserting it, we can drop ours. 241 */ 242 if (cmpxchg(&evtchn_to_irq[row], NULL, evtchn_row) != NULL) 243 free_page((unsigned long) evtchn_row); 244 } 245 246 WRITE_ONCE(evtchn_to_irq[row][col], irq); 247 return 0; 248} 249 250/* Get info for IRQ */ 251static struct irq_info *info_for_irq(unsigned irq) 252{ 253 if (irq < nr_legacy_irqs()) 254 return legacy_info_ptrs[irq]; 255 else 256 return irq_get_chip_data(irq); 257} 258 259static void set_info_for_irq(unsigned int irq, struct irq_info *info) 260{ 261 if (irq < nr_legacy_irqs()) 262 legacy_info_ptrs[irq] = info; 263 else 264 irq_set_chip_data(irq, info); 265} 266 267static struct irq_info *evtchn_to_info(evtchn_port_t evtchn) 268{ 269 int irq; 270 271 if (evtchn >= xen_evtchn_max_channels()) 272 return NULL; 273 if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL) 274 return NULL; 275 irq = READ_ONCE(evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)]); 276 277 return (irq < 0) ? NULL : info_for_irq(irq); 278} 279 280/* Per CPU channel accounting */ 281static void channels_on_cpu_dec(struct irq_info *info) 282{ 283 if (!info->is_accounted) 284 return; 285 286 info->is_accounted = 0; 287 288 if (WARN_ON_ONCE(info->cpu >= nr_cpu_ids)) 289 return; 290 291 WARN_ON_ONCE(!atomic_add_unless(&channels_on_cpu[info->cpu], -1 , 0)); 292} 293 294static void channels_on_cpu_inc(struct irq_info *info) 295{ 296 if (WARN_ON_ONCE(info->cpu >= nr_cpu_ids)) 297 return; 298 299 if (WARN_ON_ONCE(!atomic_add_unless(&channels_on_cpu[info->cpu], 1, 300 INT_MAX))) 301 return; 302 303 info->is_accounted = 1; 304} 305 306static void xen_irq_free_desc(unsigned int irq) 307{ 308 /* Legacy IRQ descriptors are managed by the arch. */ 309 if (irq >= nr_legacy_irqs()) 310 irq_free_desc(irq); 311} 312 313static void delayed_free_irq(struct work_struct *work) 314{ 315 struct irq_info *info = container_of(to_rcu_work(work), struct irq_info, 316 rwork); 317 unsigned int irq = info->irq; 318 319 /* Remove the info pointer only now, with no potential users left. */ 320 set_info_for_irq(irq, NULL); 321 322 kfree(info); 323 324 xen_irq_free_desc(irq); 325} 326 327/* Constructors for packed IRQ information. */ 328static int xen_irq_info_common_setup(struct irq_info *info, 329 enum xen_irq_type type, 330 evtchn_port_t evtchn, 331 unsigned short cpu) 332{ 333 int ret; 334 335 BUG_ON(info->type != IRQT_UNBOUND && info->type != type); 336 337 info->type = type; 338 info->evtchn = evtchn; 339 info->cpu = cpu; 340 info->mask_reason = EVT_MASK_REASON_EXPLICIT; 341 raw_spin_lock_init(&info->lock); 342 343 ret = set_evtchn_to_irq(evtchn, info->irq); 344 if (ret < 0) 345 return ret; 346 347 irq_clear_status_flags(info->irq, IRQ_NOREQUEST | IRQ_NOAUTOEN); 348 349 return xen_evtchn_port_setup(evtchn); 350} 351 352static int xen_irq_info_evtchn_setup(struct irq_info *info, 353 evtchn_port_t evtchn, 354 struct xenbus_device *dev) 355{ 356 int ret; 357 358 ret = xen_irq_info_common_setup(info, IRQT_EVTCHN, evtchn, 0); 359 info->u.interdomain = dev; 360 if (dev) 361 atomic_inc(&dev->event_channels); 362 363 return ret; 364} 365 366static int xen_irq_info_ipi_setup(struct irq_info *info, unsigned int cpu, 367 evtchn_port_t evtchn, enum ipi_vector ipi) 368{ 369 info->u.ipi = ipi; 370 371 per_cpu(ipi_to_irq, cpu)[ipi] = info->irq; 372 per_cpu(ipi_to_evtchn, cpu)[ipi] = evtchn; 373 374 return xen_irq_info_common_setup(info, IRQT_IPI, evtchn, 0); 375} 376 377static int xen_irq_info_virq_setup(struct irq_info *info, unsigned int cpu, 378 evtchn_port_t evtchn, unsigned int virq) 379{ 380 info->u.virq = virq; 381 382 per_cpu(virq_to_irq, cpu)[virq] = info->irq; 383 384 return xen_irq_info_common_setup(info, IRQT_VIRQ, evtchn, 0); 385} 386 387static int xen_irq_info_pirq_setup(struct irq_info *info, evtchn_port_t evtchn, 388 unsigned int pirq, unsigned int gsi, 389 uint16_t domid, unsigned char flags) 390{ 391 info->u.pirq.pirq = pirq; 392 info->u.pirq.gsi = gsi; 393 info->u.pirq.domid = domid; 394 info->u.pirq.flags = flags; 395 396 return xen_irq_info_common_setup(info, IRQT_PIRQ, evtchn, 0); 397} 398 399static void xen_irq_info_cleanup(struct irq_info *info) 400{ 401 set_evtchn_to_irq(info->evtchn, -1); 402 xen_evtchn_port_remove(info->evtchn, info->cpu); 403 info->evtchn = 0; 404 channels_on_cpu_dec(info); 405} 406 407/* 408 * Accessors for packed IRQ information. 409 */ 410static evtchn_port_t evtchn_from_irq(unsigned int irq) 411{ 412 const struct irq_info *info = NULL; 413 414 if (likely(irq < irq_get_nr_irqs())) 415 info = info_for_irq(irq); 416 if (!info) 417 return 0; 418 419 return info->evtchn; 420} 421 422unsigned int irq_from_evtchn(evtchn_port_t evtchn) 423{ 424 struct irq_info *info = evtchn_to_info(evtchn); 425 426 return info ? info->irq : -1; 427} 428EXPORT_SYMBOL_GPL(irq_from_evtchn); 429 430int irq_evtchn_from_virq(unsigned int cpu, unsigned int virq, 431 evtchn_port_t *evtchn) 432{ 433 int irq = per_cpu(virq_to_irq, cpu)[virq]; 434 435 *evtchn = evtchn_from_irq(irq); 436 437 return irq; 438} 439 440static enum ipi_vector ipi_from_irq(struct irq_info *info) 441{ 442 BUG_ON(info == NULL); 443 BUG_ON(info->type != IRQT_IPI); 444 445 return info->u.ipi; 446} 447 448static unsigned int virq_from_irq(struct irq_info *info) 449{ 450 BUG_ON(info == NULL); 451 BUG_ON(info->type != IRQT_VIRQ); 452 453 return info->u.virq; 454} 455 456static unsigned int pirq_from_irq(struct irq_info *info) 457{ 458 BUG_ON(info == NULL); 459 BUG_ON(info->type != IRQT_PIRQ); 460 461 return info->u.pirq.pirq; 462} 463 464unsigned int cpu_from_evtchn(evtchn_port_t evtchn) 465{ 466 struct irq_info *info = evtchn_to_info(evtchn); 467 468 return info ? info->cpu : 0; 469} 470 471static void do_mask(struct irq_info *info, u8 reason) 472{ 473 unsigned long flags; 474 475 raw_spin_lock_irqsave(&info->lock, flags); 476 477 if (!info->mask_reason) 478 mask_evtchn(info->evtchn); 479 480 info->mask_reason |= reason; 481 482 raw_spin_unlock_irqrestore(&info->lock, flags); 483} 484 485static void do_unmask(struct irq_info *info, u8 reason) 486{ 487 unsigned long flags; 488 489 raw_spin_lock_irqsave(&info->lock, flags); 490 491 info->mask_reason &= ~reason; 492 493 if (!info->mask_reason) 494 unmask_evtchn(info->evtchn); 495 496 raw_spin_unlock_irqrestore(&info->lock, flags); 497} 498 499#ifdef CONFIG_X86 500static bool pirq_check_eoi_map(struct irq_info *info) 501{ 502 return test_bit(pirq_from_irq(info), pirq_eoi_map); 503} 504#endif 505 506static bool pirq_needs_eoi_flag(struct irq_info *info) 507{ 508 BUG_ON(info->type != IRQT_PIRQ); 509 510 return info->u.pirq.flags & PIRQ_NEEDS_EOI; 511} 512 513static void bind_evtchn_to_cpu(struct irq_info *info, unsigned int cpu, 514 bool force_affinity) 515{ 516 if (IS_ENABLED(CONFIG_SMP) && force_affinity) { 517 struct irq_data *data = irq_get_irq_data(info->irq); 518 519 irq_data_update_affinity(data, cpumask_of(cpu)); 520 irq_data_update_effective_affinity(data, cpumask_of(cpu)); 521 } 522 523 xen_evtchn_port_bind_to_cpu(info->evtchn, cpu, info->cpu); 524 525 channels_on_cpu_dec(info); 526 info->cpu = cpu; 527 channels_on_cpu_inc(info); 528} 529 530/** 531 * notify_remote_via_irq - send event to remote end of event channel via irq 532 * @irq: irq of event channel to send event to 533 * 534 * Unlike notify_remote_via_evtchn(), this is safe to use across 535 * save/restore. Notifications on a broken connection are silently 536 * dropped. 537 */ 538void notify_remote_via_irq(int irq) 539{ 540 evtchn_port_t evtchn = evtchn_from_irq(irq); 541 542 if (VALID_EVTCHN(evtchn)) 543 notify_remote_via_evtchn(evtchn); 544} 545EXPORT_SYMBOL_GPL(notify_remote_via_irq); 546 547struct lateeoi_work { 548 struct delayed_work delayed; 549 spinlock_t eoi_list_lock; 550 struct list_head eoi_list; 551}; 552 553static DEFINE_PER_CPU(struct lateeoi_work, lateeoi); 554 555static void lateeoi_list_del(struct irq_info *info) 556{ 557 struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu); 558 unsigned long flags; 559 560 spin_lock_irqsave(&eoi->eoi_list_lock, flags); 561 list_del_init(&info->eoi_list); 562 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); 563} 564 565static void lateeoi_list_add(struct irq_info *info) 566{ 567 struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu); 568 struct irq_info *elem; 569 u64 now = get_jiffies_64(); 570 unsigned long delay; 571 unsigned long flags; 572 573 if (now < info->eoi_time) 574 delay = info->eoi_time - now; 575 else 576 delay = 1; 577 578 spin_lock_irqsave(&eoi->eoi_list_lock, flags); 579 580 elem = list_first_entry_or_null(&eoi->eoi_list, struct irq_info, 581 eoi_list); 582 if (!elem || info->eoi_time < elem->eoi_time) { 583 list_add(&info->eoi_list, &eoi->eoi_list); 584 mod_delayed_work_on(info->eoi_cpu, system_wq, 585 &eoi->delayed, delay); 586 } else { 587 list_for_each_entry_reverse(elem, &eoi->eoi_list, eoi_list) { 588 if (elem->eoi_time <= info->eoi_time) 589 break; 590 } 591 list_add(&info->eoi_list, &elem->eoi_list); 592 } 593 594 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); 595} 596 597static void xen_irq_lateeoi_locked(struct irq_info *info, bool spurious) 598{ 599 evtchn_port_t evtchn; 600 unsigned int cpu; 601 unsigned int delay = 0; 602 603 evtchn = info->evtchn; 604 if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list)) 605 return; 606 607 if (spurious) { 608 struct xenbus_device *dev = info->u.interdomain; 609 unsigned int threshold = 1; 610 611 if (dev && dev->spurious_threshold) 612 threshold = dev->spurious_threshold; 613 614 if ((1 << info->spurious_cnt) < (HZ << 2)) { 615 if (info->spurious_cnt != 0xFF) 616 info->spurious_cnt++; 617 } 618 if (info->spurious_cnt > threshold) { 619 delay = 1 << (info->spurious_cnt - 1 - threshold); 620 if (delay > HZ) 621 delay = HZ; 622 if (!info->eoi_time) 623 info->eoi_cpu = smp_processor_id(); 624 info->eoi_time = get_jiffies_64() + delay; 625 if (dev) 626 atomic_add(delay, &dev->jiffies_eoi_delayed); 627 } 628 if (dev) 629 atomic_inc(&dev->spurious_events); 630 } else { 631 info->spurious_cnt = 0; 632 } 633 634 cpu = info->eoi_cpu; 635 if (info->eoi_time && 636 (info->irq_epoch == per_cpu(irq_epoch, cpu) || delay)) { 637 lateeoi_list_add(info); 638 return; 639 } 640 641 info->eoi_time = 0; 642 643 /* is_active hasn't been reset yet, do it now. */ 644 smp_store_release(&info->is_active, 0); 645 do_unmask(info, EVT_MASK_REASON_EOI_PENDING); 646} 647 648static void xen_irq_lateeoi_worker(struct work_struct *work) 649{ 650 struct lateeoi_work *eoi; 651 struct irq_info *info; 652 u64 now = get_jiffies_64(); 653 unsigned long flags; 654 655 eoi = container_of(to_delayed_work(work), struct lateeoi_work, delayed); 656 657 rcu_read_lock(); 658 659 while (true) { 660 spin_lock_irqsave(&eoi->eoi_list_lock, flags); 661 662 info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info, 663 eoi_list); 664 665 if (info == NULL) 666 break; 667 668 if (now < info->eoi_time) { 669 mod_delayed_work_on(info->eoi_cpu, system_wq, 670 &eoi->delayed, 671 info->eoi_time - now); 672 break; 673 } 674 675 list_del_init(&info->eoi_list); 676 677 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); 678 679 info->eoi_time = 0; 680 681 xen_irq_lateeoi_locked(info, false); 682 } 683 684 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); 685 686 rcu_read_unlock(); 687} 688 689static void xen_cpu_init_eoi(unsigned int cpu) 690{ 691 struct lateeoi_work *eoi = &per_cpu(lateeoi, cpu); 692 693 INIT_DELAYED_WORK(&eoi->delayed, xen_irq_lateeoi_worker); 694 spin_lock_init(&eoi->eoi_list_lock); 695 INIT_LIST_HEAD(&eoi->eoi_list); 696} 697 698void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags) 699{ 700 struct irq_info *info; 701 702 rcu_read_lock(); 703 704 info = info_for_irq(irq); 705 706 if (info) 707 xen_irq_lateeoi_locked(info, eoi_flags & XEN_EOI_FLAG_SPURIOUS); 708 709 rcu_read_unlock(); 710} 711EXPORT_SYMBOL_GPL(xen_irq_lateeoi); 712 713static struct irq_info *xen_irq_init(unsigned int irq) 714{ 715 struct irq_info *info; 716 717 info = kzalloc(sizeof(*info), GFP_KERNEL); 718 if (info) { 719 info->irq = irq; 720 info->type = IRQT_UNBOUND; 721 info->refcnt = -1; 722 INIT_RCU_WORK(&info->rwork, delayed_free_irq); 723 724 set_info_for_irq(irq, info); 725 INIT_LIST_HEAD(&info->eoi_list); 726 list_add_tail(&info->list, &xen_irq_list_head); 727 } 728 729 return info; 730} 731 732static struct irq_info *xen_allocate_irq_dynamic(void) 733{ 734 int irq = irq_alloc_desc_from(0, -1); 735 struct irq_info *info = NULL; 736 737 if (irq >= 0) { 738 info = xen_irq_init(irq); 739 if (!info) 740 xen_irq_free_desc(irq); 741 } 742 743 return info; 744} 745 746static struct irq_info *xen_allocate_irq_gsi(unsigned int gsi) 747{ 748 int irq; 749 struct irq_info *info; 750 751 /* 752 * A PV guest has no concept of a GSI (since it has no ACPI 753 * nor access to/knowledge of the physical APICs). Therefore 754 * all IRQs are dynamically allocated from the entire IRQ 755 * space. 756 */ 757 if (xen_pv_domain() && !xen_initial_domain()) 758 return xen_allocate_irq_dynamic(); 759 760 /* Legacy IRQ descriptors are already allocated by the arch. */ 761 if (gsi < nr_legacy_irqs()) 762 irq = gsi; 763 else 764 irq = irq_alloc_desc_at(gsi, -1); 765 766 info = xen_irq_init(irq); 767 if (!info) 768 xen_irq_free_desc(irq); 769 770 return info; 771} 772 773static void xen_free_irq(struct irq_info *info) 774{ 775 if (WARN_ON(!info)) 776 return; 777 778 if (!list_empty(&info->eoi_list)) 779 lateeoi_list_del(info); 780 781 list_del(&info->list); 782 783 WARN_ON(info->refcnt > 0); 784 785 queue_rcu_work(system_wq, &info->rwork); 786} 787 788/* Not called for lateeoi events. */ 789static void event_handler_exit(struct irq_info *info) 790{ 791 smp_store_release(&info->is_active, 0); 792 clear_evtchn(info->evtchn); 793} 794 795static void pirq_query_unmask(struct irq_info *info) 796{ 797 struct physdev_irq_status_query irq_status; 798 799 irq_status.irq = pirq_from_irq(info); 800 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status)) 801 irq_status.flags = 0; 802 803 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI; 804 if (irq_status.flags & XENIRQSTAT_needs_eoi) 805 info->u.pirq.flags |= PIRQ_NEEDS_EOI; 806} 807 808static void do_eoi_pirq(struct irq_info *info) 809{ 810 struct physdev_eoi eoi = { .irq = pirq_from_irq(info) }; 811 int rc = 0; 812 813 if (!VALID_EVTCHN(info->evtchn)) 814 return; 815 816 event_handler_exit(info); 817 818 if (pirq_needs_eoi(info)) { 819 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi); 820 WARN_ON(rc); 821 } 822} 823 824static void eoi_pirq(struct irq_data *data) 825{ 826 struct irq_info *info = info_for_irq(data->irq); 827 828 do_eoi_pirq(info); 829} 830 831static void do_disable_dynirq(struct irq_info *info) 832{ 833 if (VALID_EVTCHN(info->evtchn)) 834 do_mask(info, EVT_MASK_REASON_EXPLICIT); 835} 836 837static void disable_dynirq(struct irq_data *data) 838{ 839 struct irq_info *info = info_for_irq(data->irq); 840 841 if (info) 842 do_disable_dynirq(info); 843} 844 845static void mask_ack_pirq(struct irq_data *data) 846{ 847 struct irq_info *info = info_for_irq(data->irq); 848 849 if (info) { 850 do_disable_dynirq(info); 851 do_eoi_pirq(info); 852 } 853} 854 855static unsigned int __startup_pirq(struct irq_info *info) 856{ 857 struct evtchn_bind_pirq bind_pirq; 858 evtchn_port_t evtchn = info->evtchn; 859 int rc; 860 861 if (VALID_EVTCHN(evtchn)) 862 goto out; 863 864 bind_pirq.pirq = pirq_from_irq(info); 865 /* NB. We are happy to share unless we are probing. */ 866 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ? 867 BIND_PIRQ__WILL_SHARE : 0; 868 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq); 869 if (rc != 0) { 870 pr_warn("Failed to obtain physical IRQ %d\n", info->irq); 871 return 0; 872 } 873 evtchn = bind_pirq.port; 874 875 pirq_query_unmask(info); 876 877 rc = set_evtchn_to_irq(evtchn, info->irq); 878 if (rc) 879 goto err; 880 881 info->evtchn = evtchn; 882 bind_evtchn_to_cpu(info, 0, false); 883 884 rc = xen_evtchn_port_setup(evtchn); 885 if (rc) 886 goto err; 887 888out: 889 do_unmask(info, EVT_MASK_REASON_EXPLICIT); 890 891 do_eoi_pirq(info); 892 893 return 0; 894 895err: 896 pr_err("irq%d: Failed to set port to irq mapping (%d)\n", info->irq, 897 rc); 898 xen_evtchn_close(evtchn); 899 return 0; 900} 901 902static unsigned int startup_pirq(struct irq_data *data) 903{ 904 struct irq_info *info = info_for_irq(data->irq); 905 906 return __startup_pirq(info); 907} 908 909static void shutdown_pirq(struct irq_data *data) 910{ 911 struct irq_info *info = info_for_irq(data->irq); 912 evtchn_port_t evtchn = info->evtchn; 913 914 BUG_ON(info->type != IRQT_PIRQ); 915 916 if (!VALID_EVTCHN(evtchn)) 917 return; 918 919 do_mask(info, EVT_MASK_REASON_EXPLICIT); 920 xen_irq_info_cleanup(info); 921 xen_evtchn_close(evtchn); 922} 923 924static void enable_pirq(struct irq_data *data) 925{ 926 enable_dynirq(data); 927} 928 929static void disable_pirq(struct irq_data *data) 930{ 931 disable_dynirq(data); 932} 933 934int xen_irq_from_gsi(unsigned gsi) 935{ 936 struct irq_info *info; 937 938 list_for_each_entry(info, &xen_irq_list_head, list) { 939 if (info->type != IRQT_PIRQ) 940 continue; 941 942 if (info->u.pirq.gsi == gsi) 943 return info->irq; 944 } 945 946 return -1; 947} 948EXPORT_SYMBOL_GPL(xen_irq_from_gsi); 949 950static void __unbind_from_irq(struct irq_info *info, unsigned int irq) 951{ 952 evtchn_port_t evtchn; 953 bool close_evtchn = false; 954 955 if (!info) { 956 xen_irq_free_desc(irq); 957 return; 958 } 959 960 if (info->refcnt > 0) { 961 info->refcnt--; 962 if (info->refcnt != 0) 963 return; 964 } 965 966 evtchn = info->evtchn; 967 968 if (VALID_EVTCHN(evtchn)) { 969 unsigned int cpu = info->cpu; 970 struct xenbus_device *dev; 971 972 if (!info->is_static) 973 close_evtchn = true; 974 975 switch (info->type) { 976 case IRQT_VIRQ: 977 per_cpu(virq_to_irq, cpu)[virq_from_irq(info)] = -1; 978 break; 979 case IRQT_IPI: 980 per_cpu(ipi_to_irq, cpu)[ipi_from_irq(info)] = -1; 981 per_cpu(ipi_to_evtchn, cpu)[ipi_from_irq(info)] = 0; 982 break; 983 case IRQT_EVTCHN: 984 dev = info->u.interdomain; 985 if (dev) 986 atomic_dec(&dev->event_channels); 987 break; 988 default: 989 break; 990 } 991 992 xen_irq_info_cleanup(info); 993 994 if (close_evtchn) 995 xen_evtchn_close(evtchn); 996 } 997 998 xen_free_irq(info); 999} 1000 1001/* 1002 * Do not make any assumptions regarding the relationship between the 1003 * IRQ number returned here and the Xen pirq argument. 1004 * 1005 * Note: We don't assign an event channel until the irq actually started 1006 * up. Return an existing irq if we've already got one for the gsi. 1007 * 1008 * Shareable implies level triggered, not shareable implies edge 1009 * triggered here. 1010 */ 1011int xen_bind_pirq_gsi_to_irq(unsigned gsi, 1012 unsigned pirq, int shareable, char *name) 1013{ 1014 struct irq_info *info; 1015 struct physdev_irq irq_op; 1016 int ret; 1017 1018 mutex_lock(&irq_mapping_update_lock); 1019 1020 ret = xen_irq_from_gsi(gsi); 1021 if (ret != -1) { 1022 pr_info("%s: returning irq %d for gsi %u\n", 1023 __func__, ret, gsi); 1024 goto out; 1025 } 1026 1027 info = xen_allocate_irq_gsi(gsi); 1028 if (!info) 1029 goto out; 1030 1031 irq_op.irq = info->irq; 1032 irq_op.vector = 0; 1033 1034 /* Only the privileged domain can do this. For non-priv, the pcifront 1035 * driver provides a PCI bus that does the call to do exactly 1036 * this in the priv domain. */ 1037 if (xen_initial_domain() && 1038 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) { 1039 xen_free_irq(info); 1040 ret = -ENOSPC; 1041 goto out; 1042 } 1043 1044 ret = xen_irq_info_pirq_setup(info, 0, pirq, gsi, DOMID_SELF, 1045 shareable ? PIRQ_SHAREABLE : 0); 1046 if (ret < 0) { 1047 __unbind_from_irq(info, info->irq); 1048 goto out; 1049 } 1050 1051 pirq_query_unmask(info); 1052 /* We try to use the handler with the appropriate semantic for the 1053 * type of interrupt: if the interrupt is an edge triggered 1054 * interrupt we use handle_edge_irq. 1055 * 1056 * On the other hand if the interrupt is level triggered we use 1057 * handle_fasteoi_irq like the native code does for this kind of 1058 * interrupts. 1059 * 1060 * Depending on the Xen version, pirq_needs_eoi might return true 1061 * not only for level triggered interrupts but for edge triggered 1062 * interrupts too. In any case Xen always honors the eoi mechanism, 1063 * not injecting any more pirqs of the same kind if the first one 1064 * hasn't received an eoi yet. Therefore using the fasteoi handler 1065 * is the right choice either way. 1066 */ 1067 if (shareable) 1068 irq_set_chip_and_handler_name(info->irq, &xen_pirq_chip, 1069 handle_fasteoi_irq, name); 1070 else 1071 irq_set_chip_and_handler_name(info->irq, &xen_pirq_chip, 1072 handle_edge_irq, name); 1073 1074 ret = info->irq; 1075 1076out: 1077 mutex_unlock(&irq_mapping_update_lock); 1078 1079 return ret; 1080} 1081 1082#ifdef CONFIG_PCI_MSI 1083int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc) 1084{ 1085 int rc; 1086 struct physdev_get_free_pirq op_get_free_pirq; 1087 1088 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI; 1089 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq); 1090 1091 WARN_ONCE(rc == -ENOSYS, 1092 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n"); 1093 1094 return rc ? -1 : op_get_free_pirq.pirq; 1095} 1096 1097int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc, 1098 int pirq, int nvec, const char *name, domid_t domid) 1099{ 1100 int i, irq, ret; 1101 struct irq_info *info; 1102 1103 mutex_lock(&irq_mapping_update_lock); 1104 1105 irq = irq_alloc_descs(-1, 0, nvec, -1); 1106 if (irq < 0) 1107 goto out; 1108 1109 for (i = 0; i < nvec; i++) { 1110 info = xen_irq_init(irq + i); 1111 if (!info) { 1112 ret = -ENOMEM; 1113 goto error_irq; 1114 } 1115 1116 irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name); 1117 1118 ret = xen_irq_info_pirq_setup(info, 0, pirq + i, 0, domid, 1119 i == 0 ? 0 : PIRQ_MSI_GROUP); 1120 if (ret < 0) 1121 goto error_irq; 1122 } 1123 1124 ret = irq_set_msi_desc(irq, msidesc); 1125 if (ret < 0) 1126 goto error_irq; 1127out: 1128 mutex_unlock(&irq_mapping_update_lock); 1129 return irq; 1130 1131error_irq: 1132 while (nvec--) { 1133 info = info_for_irq(irq + nvec); 1134 __unbind_from_irq(info, irq + nvec); 1135 } 1136 mutex_unlock(&irq_mapping_update_lock); 1137 return ret; 1138} 1139#endif 1140 1141int xen_destroy_irq(int irq) 1142{ 1143 struct physdev_unmap_pirq unmap_irq; 1144 struct irq_info *info = info_for_irq(irq); 1145 int rc = -ENOENT; 1146 1147 mutex_lock(&irq_mapping_update_lock); 1148 1149 /* 1150 * If trying to remove a vector in a MSI group different 1151 * than the first one skip the PIRQ unmap unless this vector 1152 * is the first one in the group. 1153 */ 1154 if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) { 1155 unmap_irq.pirq = info->u.pirq.pirq; 1156 unmap_irq.domid = info->u.pirq.domid; 1157 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq); 1158 /* If another domain quits without making the pci_disable_msix 1159 * call, the Xen hypervisor takes care of freeing the PIRQs 1160 * (free_domain_pirqs). 1161 */ 1162 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF)) 1163 pr_info("domain %d does not have %d anymore\n", 1164 info->u.pirq.domid, info->u.pirq.pirq); 1165 else if (rc) { 1166 pr_warn("unmap irq failed %d\n", rc); 1167 goto out; 1168 } 1169 } 1170 1171 xen_free_irq(info); 1172 1173out: 1174 mutex_unlock(&irq_mapping_update_lock); 1175 return rc; 1176} 1177 1178int xen_pirq_from_irq(unsigned irq) 1179{ 1180 struct irq_info *info = info_for_irq(irq); 1181 1182 return pirq_from_irq(info); 1183} 1184EXPORT_SYMBOL_GPL(xen_pirq_from_irq); 1185 1186static int bind_evtchn_to_irq_chip(evtchn_port_t evtchn, struct irq_chip *chip, 1187 struct xenbus_device *dev, bool shared) 1188{ 1189 int ret = -ENOMEM; 1190 struct irq_info *info; 1191 1192 if (evtchn >= xen_evtchn_max_channels()) 1193 return -ENOMEM; 1194 1195 mutex_lock(&irq_mapping_update_lock); 1196 1197 info = evtchn_to_info(evtchn); 1198 1199 if (!info) { 1200 info = xen_allocate_irq_dynamic(); 1201 if (!info) 1202 goto out; 1203 1204 irq_set_chip_and_handler_name(info->irq, chip, 1205 handle_edge_irq, "event"); 1206 1207 ret = xen_irq_info_evtchn_setup(info, evtchn, dev); 1208 if (ret < 0) { 1209 __unbind_from_irq(info, info->irq); 1210 goto out; 1211 } 1212 /* 1213 * New interdomain events are initially bound to vCPU0 This 1214 * is required to setup the event channel in the first 1215 * place and also important for UP guests because the 1216 * affinity setting is not invoked on them so nothing would 1217 * bind the channel. 1218 */ 1219 bind_evtchn_to_cpu(info, 0, false); 1220 } else if (!WARN_ON(info->type != IRQT_EVTCHN)) { 1221 if (shared && !WARN_ON(info->refcnt < 0)) 1222 info->refcnt++; 1223 } 1224 1225 ret = info->irq; 1226 1227out: 1228 mutex_unlock(&irq_mapping_update_lock); 1229 1230 return ret; 1231} 1232 1233int bind_evtchn_to_irq(evtchn_port_t evtchn) 1234{ 1235 return bind_evtchn_to_irq_chip(evtchn, &xen_dynamic_chip, NULL, false); 1236} 1237EXPORT_SYMBOL_GPL(bind_evtchn_to_irq); 1238 1239int bind_evtchn_to_irq_lateeoi(evtchn_port_t evtchn) 1240{ 1241 return bind_evtchn_to_irq_chip(evtchn, &xen_lateeoi_chip, NULL, false); 1242} 1243EXPORT_SYMBOL_GPL(bind_evtchn_to_irq_lateeoi); 1244 1245static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu) 1246{ 1247 struct evtchn_bind_ipi bind_ipi; 1248 evtchn_port_t evtchn; 1249 struct irq_info *info; 1250 int ret; 1251 1252 mutex_lock(&irq_mapping_update_lock); 1253 1254 ret = per_cpu(ipi_to_irq, cpu)[ipi]; 1255 1256 if (ret == -1) { 1257 info = xen_allocate_irq_dynamic(); 1258 if (!info) 1259 goto out; 1260 1261 irq_set_chip_and_handler_name(info->irq, &xen_percpu_chip, 1262 handle_percpu_irq, "ipi"); 1263 1264 bind_ipi.vcpu = xen_vcpu_nr(cpu); 1265 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 1266 &bind_ipi) != 0) 1267 BUG(); 1268 evtchn = bind_ipi.port; 1269 1270 ret = xen_irq_info_ipi_setup(info, cpu, evtchn, ipi); 1271 if (ret < 0) { 1272 __unbind_from_irq(info, info->irq); 1273 goto out; 1274 } 1275 /* 1276 * Force the affinity mask to the target CPU so proc shows 1277 * the correct target. 1278 */ 1279 bind_evtchn_to_cpu(info, cpu, true); 1280 ret = info->irq; 1281 } else { 1282 info = info_for_irq(ret); 1283 WARN_ON(info == NULL || info->type != IRQT_IPI); 1284 } 1285 1286 out: 1287 mutex_unlock(&irq_mapping_update_lock); 1288 return ret; 1289} 1290 1291static int bind_interdomain_evtchn_to_irq_chip(struct xenbus_device *dev, 1292 evtchn_port_t remote_port, 1293 struct irq_chip *chip, 1294 bool shared) 1295{ 1296 struct evtchn_bind_interdomain bind_interdomain; 1297 int err; 1298 1299 bind_interdomain.remote_dom = dev->otherend_id; 1300 bind_interdomain.remote_port = remote_port; 1301 1302 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, 1303 &bind_interdomain); 1304 1305 return err ? : bind_evtchn_to_irq_chip(bind_interdomain.local_port, 1306 chip, dev, shared); 1307} 1308 1309int bind_interdomain_evtchn_to_irq_lateeoi(struct xenbus_device *dev, 1310 evtchn_port_t remote_port) 1311{ 1312 return bind_interdomain_evtchn_to_irq_chip(dev, remote_port, 1313 &xen_lateeoi_chip, false); 1314} 1315EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq_lateeoi); 1316 1317static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn, 1318 bool percpu) 1319{ 1320 struct evtchn_status status; 1321 evtchn_port_t port; 1322 bool exists = false; 1323 1324 memset(&status, 0, sizeof(status)); 1325 for (port = 0; port < xen_evtchn_max_channels(); port++) { 1326 int rc; 1327 1328 status.dom = DOMID_SELF; 1329 status.port = port; 1330 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status); 1331 if (rc < 0) 1332 continue; 1333 if (status.status != EVTCHNSTAT_virq) 1334 continue; 1335 if (status.u.virq != virq) 1336 continue; 1337 if (status.vcpu == xen_vcpu_nr(cpu)) { 1338 *evtchn = port; 1339 return 0; 1340 } else if (!percpu) { 1341 exists = true; 1342 } 1343 } 1344 return exists ? -EEXIST : -ENOENT; 1345} 1346 1347/** 1348 * xen_evtchn_nr_channels - number of usable event channel ports 1349 * 1350 * This may be less than the maximum supported by the current 1351 * hypervisor ABI. Use xen_evtchn_max_channels() for the maximum 1352 * supported. 1353 */ 1354unsigned xen_evtchn_nr_channels(void) 1355{ 1356 return evtchn_ops->nr_channels(); 1357} 1358EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels); 1359 1360int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu) 1361{ 1362 struct evtchn_bind_virq bind_virq; 1363 evtchn_port_t evtchn = 0; 1364 struct irq_info *info; 1365 int ret; 1366 1367 mutex_lock(&irq_mapping_update_lock); 1368 1369 ret = per_cpu(virq_to_irq, cpu)[virq]; 1370 1371 if (ret == -1) { 1372 info = xen_allocate_irq_dynamic(); 1373 if (!info) 1374 goto out; 1375 1376 if (percpu) 1377 irq_set_chip_and_handler_name(info->irq, &xen_percpu_chip, 1378 handle_percpu_irq, "virq"); 1379 else 1380 irq_set_chip_and_handler_name(info->irq, &xen_dynamic_chip, 1381 handle_edge_irq, "virq"); 1382 1383 bind_virq.virq = virq; 1384 bind_virq.vcpu = xen_vcpu_nr(cpu); 1385 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 1386 &bind_virq); 1387 if (ret == 0) 1388 evtchn = bind_virq.port; 1389 else { 1390 if (ret == -EEXIST) 1391 ret = find_virq(virq, cpu, &evtchn, percpu); 1392 if (ret) { 1393 __unbind_from_irq(info, info->irq); 1394 goto out; 1395 } 1396 } 1397 1398 ret = xen_irq_info_virq_setup(info, cpu, evtchn, virq); 1399 if (ret < 0) { 1400 __unbind_from_irq(info, info->irq); 1401 goto out; 1402 } 1403 1404 /* 1405 * Force the affinity mask for percpu interrupts so proc 1406 * shows the correct target. 1407 */ 1408 bind_evtchn_to_cpu(info, cpu, percpu); 1409 ret = info->irq; 1410 } else { 1411 info = info_for_irq(ret); 1412 WARN_ON(info == NULL || info->type != IRQT_VIRQ); 1413 } 1414 1415out: 1416 mutex_unlock(&irq_mapping_update_lock); 1417 1418 return ret; 1419} 1420 1421static void unbind_from_irq(unsigned int irq) 1422{ 1423 struct irq_info *info; 1424 1425 mutex_lock(&irq_mapping_update_lock); 1426 info = info_for_irq(irq); 1427 __unbind_from_irq(info, irq); 1428 mutex_unlock(&irq_mapping_update_lock); 1429} 1430 1431static int bind_evtchn_to_irqhandler_chip(evtchn_port_t evtchn, 1432 irq_handler_t handler, 1433 unsigned long irqflags, 1434 const char *devname, void *dev_id, 1435 struct irq_chip *chip) 1436{ 1437 int irq, retval; 1438 1439 irq = bind_evtchn_to_irq_chip(evtchn, chip, NULL, 1440 irqflags & IRQF_SHARED); 1441 if (irq < 0) 1442 return irq; 1443 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1444 if (retval != 0) { 1445 unbind_from_irq(irq); 1446 return retval; 1447 } 1448 1449 return irq; 1450} 1451 1452int bind_evtchn_to_irqhandler(evtchn_port_t evtchn, 1453 irq_handler_t handler, 1454 unsigned long irqflags, 1455 const char *devname, void *dev_id) 1456{ 1457 return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags, 1458 devname, dev_id, 1459 &xen_dynamic_chip); 1460} 1461EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler); 1462 1463int bind_evtchn_to_irqhandler_lateeoi(evtchn_port_t evtchn, 1464 irq_handler_t handler, 1465 unsigned long irqflags, 1466 const char *devname, void *dev_id) 1467{ 1468 return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags, 1469 devname, dev_id, 1470 &xen_lateeoi_chip); 1471} 1472EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler_lateeoi); 1473 1474static int bind_interdomain_evtchn_to_irqhandler_chip( 1475 struct xenbus_device *dev, evtchn_port_t remote_port, 1476 irq_handler_t handler, unsigned long irqflags, 1477 const char *devname, void *dev_id, struct irq_chip *chip) 1478{ 1479 int irq, retval; 1480 1481 irq = bind_interdomain_evtchn_to_irq_chip(dev, remote_port, chip, 1482 irqflags & IRQF_SHARED); 1483 if (irq < 0) 1484 return irq; 1485 1486 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1487 if (retval != 0) { 1488 unbind_from_irq(irq); 1489 return retval; 1490 } 1491 1492 return irq; 1493} 1494 1495int bind_interdomain_evtchn_to_irqhandler_lateeoi(struct xenbus_device *dev, 1496 evtchn_port_t remote_port, 1497 irq_handler_t handler, 1498 unsigned long irqflags, 1499 const char *devname, 1500 void *dev_id) 1501{ 1502 return bind_interdomain_evtchn_to_irqhandler_chip(dev, 1503 remote_port, handler, irqflags, devname, 1504 dev_id, &xen_lateeoi_chip); 1505} 1506EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler_lateeoi); 1507 1508int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu, 1509 irq_handler_t handler, 1510 unsigned long irqflags, const char *devname, void *dev_id) 1511{ 1512 int irq, retval; 1513 1514 irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU); 1515 if (irq < 0) 1516 return irq; 1517 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1518 if (retval != 0) { 1519 unbind_from_irq(irq); 1520 return retval; 1521 } 1522 1523 return irq; 1524} 1525EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler); 1526 1527int bind_ipi_to_irqhandler(enum ipi_vector ipi, 1528 unsigned int cpu, 1529 irq_handler_t handler, 1530 unsigned long irqflags, 1531 const char *devname, 1532 void *dev_id) 1533{ 1534 int irq, retval; 1535 1536 irq = bind_ipi_to_irq(ipi, cpu); 1537 if (irq < 0) 1538 return irq; 1539 1540 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME; 1541 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1542 if (retval != 0) { 1543 unbind_from_irq(irq); 1544 return retval; 1545 } 1546 1547 return irq; 1548} 1549 1550void unbind_from_irqhandler(unsigned int irq, void *dev_id) 1551{ 1552 struct irq_info *info = info_for_irq(irq); 1553 1554 if (WARN_ON(!info)) 1555 return; 1556 free_irq(irq, dev_id); 1557 unbind_from_irq(irq); 1558} 1559EXPORT_SYMBOL_GPL(unbind_from_irqhandler); 1560 1561/** 1562 * xen_set_irq_priority() - set an event channel priority. 1563 * @irq:irq bound to an event channel. 1564 * @priority: priority between XEN_IRQ_PRIORITY_MAX and XEN_IRQ_PRIORITY_MIN. 1565 */ 1566int xen_set_irq_priority(unsigned irq, unsigned priority) 1567{ 1568 struct evtchn_set_priority set_priority; 1569 1570 set_priority.port = evtchn_from_irq(irq); 1571 set_priority.priority = priority; 1572 1573 return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority, 1574 &set_priority); 1575} 1576EXPORT_SYMBOL_GPL(xen_set_irq_priority); 1577 1578int evtchn_make_refcounted(evtchn_port_t evtchn, bool is_static) 1579{ 1580 struct irq_info *info = evtchn_to_info(evtchn); 1581 1582 if (!info) 1583 return -ENOENT; 1584 1585 WARN_ON(info->refcnt != -1); 1586 1587 info->refcnt = 1; 1588 info->is_static = is_static; 1589 1590 return 0; 1591} 1592EXPORT_SYMBOL_GPL(evtchn_make_refcounted); 1593 1594int evtchn_get(evtchn_port_t evtchn) 1595{ 1596 struct irq_info *info; 1597 int err = -ENOENT; 1598 1599 if (evtchn >= xen_evtchn_max_channels()) 1600 return -EINVAL; 1601 1602 mutex_lock(&irq_mapping_update_lock); 1603 1604 info = evtchn_to_info(evtchn); 1605 1606 if (!info) 1607 goto done; 1608 1609 err = -EINVAL; 1610 if (info->refcnt <= 0 || info->refcnt == SHRT_MAX) 1611 goto done; 1612 1613 info->refcnt++; 1614 err = 0; 1615 done: 1616 mutex_unlock(&irq_mapping_update_lock); 1617 1618 return err; 1619} 1620EXPORT_SYMBOL_GPL(evtchn_get); 1621 1622void evtchn_put(evtchn_port_t evtchn) 1623{ 1624 struct irq_info *info = evtchn_to_info(evtchn); 1625 1626 if (WARN_ON(!info)) 1627 return; 1628 unbind_from_irq(info->irq); 1629} 1630EXPORT_SYMBOL_GPL(evtchn_put); 1631 1632void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector) 1633{ 1634 evtchn_port_t evtchn; 1635 1636#ifdef CONFIG_X86 1637 if (unlikely(vector == XEN_NMI_VECTOR)) { 1638 int rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, xen_vcpu_nr(cpu), 1639 NULL); 1640 if (rc < 0) 1641 printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc); 1642 return; 1643 } 1644#endif 1645 evtchn = per_cpu(ipi_to_evtchn, cpu)[vector]; 1646 BUG_ON(evtchn == 0); 1647 notify_remote_via_evtchn(evtchn); 1648} 1649 1650struct evtchn_loop_ctrl { 1651 ktime_t timeout; 1652 unsigned count; 1653 bool defer_eoi; 1654}; 1655 1656void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl) 1657{ 1658 struct irq_info *info = evtchn_to_info(port); 1659 struct xenbus_device *dev; 1660 1661 if (!info) 1662 return; 1663 1664 /* 1665 * Check for timeout every 256 events. 1666 * We are setting the timeout value only after the first 256 1667 * events in order to not hurt the common case of few loop 1668 * iterations. The 256 is basically an arbitrary value. 1669 * 1670 * In case we are hitting the timeout we need to defer all further 1671 * EOIs in order to ensure to leave the event handling loop rather 1672 * sooner than later. 1673 */ 1674 if (!ctrl->defer_eoi && !(++ctrl->count & 0xff)) { 1675 ktime_t kt = ktime_get(); 1676 1677 if (!ctrl->timeout) { 1678 kt = ktime_add_ms(kt, 1679 jiffies_to_msecs(event_loop_timeout)); 1680 ctrl->timeout = kt; 1681 } else if (kt > ctrl->timeout) { 1682 ctrl->defer_eoi = true; 1683 } 1684 } 1685 1686 if (xchg_acquire(&info->is_active, 1)) 1687 return; 1688 1689 dev = (info->type == IRQT_EVTCHN) ? info->u.interdomain : NULL; 1690 if (dev) 1691 atomic_inc(&dev->events); 1692 1693 if (ctrl->defer_eoi) { 1694 info->eoi_cpu = smp_processor_id(); 1695 info->irq_epoch = __this_cpu_read(irq_epoch); 1696 info->eoi_time = get_jiffies_64() + event_eoi_delay; 1697 } 1698 1699 generic_handle_irq(info->irq); 1700} 1701 1702int xen_evtchn_do_upcall(void) 1703{ 1704 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu); 1705 int ret = vcpu_info->evtchn_upcall_pending ? IRQ_HANDLED : IRQ_NONE; 1706 int cpu = smp_processor_id(); 1707 struct evtchn_loop_ctrl ctrl = { 0 }; 1708 1709 /* 1710 * When closing an event channel the associated IRQ must not be freed 1711 * until all cpus have left the event handling loop. This is ensured 1712 * by taking the rcu_read_lock() while handling events, as freeing of 1713 * the IRQ is handled via queue_rcu_work() _after_ closing the event 1714 * channel. 1715 */ 1716 rcu_read_lock(); 1717 1718 do { 1719 vcpu_info->evtchn_upcall_pending = 0; 1720 1721 xen_evtchn_handle_events(cpu, &ctrl); 1722 1723 BUG_ON(!irqs_disabled()); 1724 1725 virt_rmb(); /* Hypervisor can set upcall pending. */ 1726 1727 } while (vcpu_info->evtchn_upcall_pending); 1728 1729 rcu_read_unlock(); 1730 1731 /* 1732 * Increment irq_epoch only now to defer EOIs only for 1733 * xen_irq_lateeoi() invocations occurring from inside the loop 1734 * above. 1735 */ 1736 __this_cpu_inc(irq_epoch); 1737 1738 return ret; 1739} 1740EXPORT_SYMBOL_GPL(xen_evtchn_do_upcall); 1741 1742/* Rebind a new event channel to an existing irq. */ 1743void rebind_evtchn_irq(evtchn_port_t evtchn, int irq) 1744{ 1745 struct irq_info *info = info_for_irq(irq); 1746 1747 if (WARN_ON(!info)) 1748 return; 1749 1750 /* Make sure the irq is masked, since the new event channel 1751 will also be masked. */ 1752 disable_irq(irq); 1753 1754 mutex_lock(&irq_mapping_update_lock); 1755 1756 /* After resume the irq<->evtchn mappings are all cleared out */ 1757 BUG_ON(evtchn_to_info(evtchn)); 1758 /* Expect irq to have been bound before, 1759 so there should be a proper type */ 1760 BUG_ON(info->type == IRQT_UNBOUND); 1761 1762 info->irq = irq; 1763 (void)xen_irq_info_evtchn_setup(info, evtchn, NULL); 1764 1765 mutex_unlock(&irq_mapping_update_lock); 1766 1767 bind_evtchn_to_cpu(info, info->cpu, false); 1768 1769 /* Unmask the event channel. */ 1770 enable_irq(irq); 1771} 1772 1773/* Rebind an evtchn so that it gets delivered to a specific cpu */ 1774static int xen_rebind_evtchn_to_cpu(struct irq_info *info, unsigned int tcpu) 1775{ 1776 struct evtchn_bind_vcpu bind_vcpu; 1777 evtchn_port_t evtchn = info ? info->evtchn : 0; 1778 1779 if (!VALID_EVTCHN(evtchn)) 1780 return -1; 1781 1782 if (!xen_support_evtchn_rebind()) 1783 return -1; 1784 1785 /* Send future instances of this interrupt to other vcpu. */ 1786 bind_vcpu.port = evtchn; 1787 bind_vcpu.vcpu = xen_vcpu_nr(tcpu); 1788 1789 /* 1790 * Mask the event while changing the VCPU binding to prevent 1791 * it being delivered on an unexpected VCPU. 1792 */ 1793 do_mask(info, EVT_MASK_REASON_TEMPORARY); 1794 1795 /* 1796 * If this fails, it usually just indicates that we're dealing with a 1797 * virq or IPI channel, which don't actually need to be rebound. Ignore 1798 * it, but don't do the xenlinux-level rebind in that case. 1799 */ 1800 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0) { 1801 int old_cpu = info->cpu; 1802 1803 bind_evtchn_to_cpu(info, tcpu, false); 1804 1805 if (info->type == IRQT_VIRQ) { 1806 int virq = info->u.virq; 1807 int irq = per_cpu(virq_to_irq, old_cpu)[virq]; 1808 1809 per_cpu(virq_to_irq, old_cpu)[virq] = -1; 1810 per_cpu(virq_to_irq, tcpu)[virq] = irq; 1811 } 1812 } 1813 1814 do_unmask(info, EVT_MASK_REASON_TEMPORARY); 1815 1816 return 0; 1817} 1818 1819/* 1820 * Find the CPU within @dest mask which has the least number of channels 1821 * assigned. This is not precise as the per cpu counts can be modified 1822 * concurrently. 1823 */ 1824static unsigned int select_target_cpu(const struct cpumask *dest) 1825{ 1826 unsigned int cpu, best_cpu = UINT_MAX, minch = UINT_MAX; 1827 1828 for_each_cpu_and(cpu, dest, cpu_online_mask) { 1829 unsigned int curch = atomic_read(&channels_on_cpu[cpu]); 1830 1831 if (curch < minch) { 1832 minch = curch; 1833 best_cpu = cpu; 1834 } 1835 } 1836 1837 /* 1838 * Catch the unlikely case that dest contains no online CPUs. Can't 1839 * recurse. 1840 */ 1841 if (best_cpu == UINT_MAX) 1842 return select_target_cpu(cpu_online_mask); 1843 1844 return best_cpu; 1845} 1846 1847static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest, 1848 bool force) 1849{ 1850 unsigned int tcpu = select_target_cpu(dest); 1851 int ret; 1852 1853 ret = xen_rebind_evtchn_to_cpu(info_for_irq(data->irq), tcpu); 1854 if (!ret) 1855 irq_data_update_effective_affinity(data, cpumask_of(tcpu)); 1856 1857 return ret; 1858} 1859 1860static void enable_dynirq(struct irq_data *data) 1861{ 1862 struct irq_info *info = info_for_irq(data->irq); 1863 evtchn_port_t evtchn = info ? info->evtchn : 0; 1864 1865 if (VALID_EVTCHN(evtchn)) 1866 do_unmask(info, EVT_MASK_REASON_EXPLICIT); 1867} 1868 1869static void do_ack_dynirq(struct irq_info *info) 1870{ 1871 evtchn_port_t evtchn = info->evtchn; 1872 1873 if (VALID_EVTCHN(evtchn)) 1874 event_handler_exit(info); 1875} 1876 1877static void ack_dynirq(struct irq_data *data) 1878{ 1879 struct irq_info *info = info_for_irq(data->irq); 1880 1881 if (info) 1882 do_ack_dynirq(info); 1883} 1884 1885static void mask_ack_dynirq(struct irq_data *data) 1886{ 1887 struct irq_info *info = info_for_irq(data->irq); 1888 1889 if (info) { 1890 do_disable_dynirq(info); 1891 do_ack_dynirq(info); 1892 } 1893} 1894 1895static void lateeoi_ack_dynirq(struct irq_data *data) 1896{ 1897 struct irq_info *info = info_for_irq(data->irq); 1898 evtchn_port_t evtchn = info ? info->evtchn : 0; 1899 1900 if (VALID_EVTCHN(evtchn)) { 1901 do_mask(info, EVT_MASK_REASON_EOI_PENDING); 1902 /* 1903 * Don't call event_handler_exit(). 1904 * Need to keep is_active non-zero in order to ignore re-raised 1905 * events after cpu affinity changes while a lateeoi is pending. 1906 */ 1907 clear_evtchn(evtchn); 1908 } 1909} 1910 1911static void lateeoi_mask_ack_dynirq(struct irq_data *data) 1912{ 1913 struct irq_info *info = info_for_irq(data->irq); 1914 evtchn_port_t evtchn = info ? info->evtchn : 0; 1915 1916 if (VALID_EVTCHN(evtchn)) { 1917 do_mask(info, EVT_MASK_REASON_EXPLICIT); 1918 event_handler_exit(info); 1919 } 1920} 1921 1922static int retrigger_dynirq(struct irq_data *data) 1923{ 1924 struct irq_info *info = info_for_irq(data->irq); 1925 evtchn_port_t evtchn = info ? info->evtchn : 0; 1926 1927 if (!VALID_EVTCHN(evtchn)) 1928 return 0; 1929 1930 do_mask(info, EVT_MASK_REASON_TEMPORARY); 1931 set_evtchn(evtchn); 1932 do_unmask(info, EVT_MASK_REASON_TEMPORARY); 1933 1934 return 1; 1935} 1936 1937static void restore_pirqs(void) 1938{ 1939 int pirq, rc, irq, gsi; 1940 struct physdev_map_pirq map_irq; 1941 struct irq_info *info; 1942 1943 list_for_each_entry(info, &xen_irq_list_head, list) { 1944 if (info->type != IRQT_PIRQ) 1945 continue; 1946 1947 pirq = info->u.pirq.pirq; 1948 gsi = info->u.pirq.gsi; 1949 irq = info->irq; 1950 1951 /* save/restore of PT devices doesn't work, so at this point the 1952 * only devices present are GSI based emulated devices */ 1953 if (!gsi) 1954 continue; 1955 1956 map_irq.domid = DOMID_SELF; 1957 map_irq.type = MAP_PIRQ_TYPE_GSI; 1958 map_irq.index = gsi; 1959 map_irq.pirq = pirq; 1960 1961 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq); 1962 if (rc) { 1963 pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n", 1964 gsi, irq, pirq, rc); 1965 xen_free_irq(info); 1966 continue; 1967 } 1968 1969 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq); 1970 1971 __startup_pirq(info); 1972 } 1973} 1974 1975static void restore_cpu_virqs(unsigned int cpu) 1976{ 1977 struct evtchn_bind_virq bind_virq; 1978 evtchn_port_t evtchn; 1979 struct irq_info *info; 1980 int virq, irq; 1981 1982 for (virq = 0; virq < NR_VIRQS; virq++) { 1983 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1) 1984 continue; 1985 info = info_for_irq(irq); 1986 1987 BUG_ON(virq_from_irq(info) != virq); 1988 1989 /* Get a new binding from Xen. */ 1990 bind_virq.virq = virq; 1991 bind_virq.vcpu = xen_vcpu_nr(cpu); 1992 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 1993 &bind_virq) != 0) 1994 BUG(); 1995 evtchn = bind_virq.port; 1996 1997 /* Record the new mapping. */ 1998 xen_irq_info_virq_setup(info, cpu, evtchn, virq); 1999 /* The affinity mask is still valid */ 2000 bind_evtchn_to_cpu(info, cpu, false); 2001 } 2002} 2003 2004static void restore_cpu_ipis(unsigned int cpu) 2005{ 2006 struct evtchn_bind_ipi bind_ipi; 2007 evtchn_port_t evtchn; 2008 struct irq_info *info; 2009 int ipi, irq; 2010 2011 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) { 2012 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1) 2013 continue; 2014 info = info_for_irq(irq); 2015 2016 BUG_ON(ipi_from_irq(info) != ipi); 2017 2018 /* Get a new binding from Xen. */ 2019 bind_ipi.vcpu = xen_vcpu_nr(cpu); 2020 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 2021 &bind_ipi) != 0) 2022 BUG(); 2023 evtchn = bind_ipi.port; 2024 2025 /* Record the new mapping. */ 2026 xen_irq_info_ipi_setup(info, cpu, evtchn, ipi); 2027 /* The affinity mask is still valid */ 2028 bind_evtchn_to_cpu(info, cpu, false); 2029 } 2030} 2031 2032/* Clear an irq's pending state, in preparation for polling on it */ 2033void xen_clear_irq_pending(int irq) 2034{ 2035 struct irq_info *info = info_for_irq(irq); 2036 evtchn_port_t evtchn = info ? info->evtchn : 0; 2037 2038 if (VALID_EVTCHN(evtchn)) 2039 event_handler_exit(info); 2040} 2041EXPORT_SYMBOL(xen_clear_irq_pending); 2042 2043bool xen_test_irq_pending(int irq) 2044{ 2045 evtchn_port_t evtchn = evtchn_from_irq(irq); 2046 bool ret = false; 2047 2048 if (VALID_EVTCHN(evtchn)) 2049 ret = test_evtchn(evtchn); 2050 2051 return ret; 2052} 2053 2054/* Poll waiting for an irq to become pending with timeout. In the usual case, 2055 * the irq will be disabled so it won't deliver an interrupt. */ 2056void xen_poll_irq_timeout(int irq, u64 timeout) 2057{ 2058 evtchn_port_t evtchn = evtchn_from_irq(irq); 2059 2060 if (VALID_EVTCHN(evtchn)) { 2061 struct sched_poll poll; 2062 2063 poll.nr_ports = 1; 2064 poll.timeout = timeout; 2065 set_xen_guest_handle(poll.ports, &evtchn); 2066 2067 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0) 2068 BUG(); 2069 } 2070} 2071EXPORT_SYMBOL(xen_poll_irq_timeout); 2072/* Poll waiting for an irq to become pending. In the usual case, the 2073 * irq will be disabled so it won't deliver an interrupt. */ 2074void xen_poll_irq(int irq) 2075{ 2076 xen_poll_irq_timeout(irq, 0 /* no timeout */); 2077} 2078 2079/* Check whether the IRQ line is shared with other guests. */ 2080int xen_test_irq_shared(int irq) 2081{ 2082 struct irq_info *info = info_for_irq(irq); 2083 struct physdev_irq_status_query irq_status; 2084 2085 if (WARN_ON(!info)) 2086 return -ENOENT; 2087 2088 irq_status.irq = info->u.pirq.pirq; 2089 2090 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status)) 2091 return 0; 2092 return !(irq_status.flags & XENIRQSTAT_shared); 2093} 2094EXPORT_SYMBOL_GPL(xen_test_irq_shared); 2095 2096void xen_irq_resume(void) 2097{ 2098 unsigned int cpu; 2099 struct irq_info *info; 2100 2101 /* New event-channel space is not 'live' yet. */ 2102 xen_evtchn_resume(); 2103 2104 /* No IRQ <-> event-channel mappings. */ 2105 list_for_each_entry(info, &xen_irq_list_head, list) { 2106 /* Zap event-channel binding */ 2107 info->evtchn = 0; 2108 /* Adjust accounting */ 2109 channels_on_cpu_dec(info); 2110 } 2111 2112 clear_evtchn_to_irq_all(); 2113 2114 for_each_possible_cpu(cpu) { 2115 restore_cpu_virqs(cpu); 2116 restore_cpu_ipis(cpu); 2117 } 2118 2119 restore_pirqs(); 2120} 2121 2122static struct irq_chip xen_dynamic_chip __read_mostly = { 2123 .name = "xen-dyn", 2124 2125 .irq_disable = disable_dynirq, 2126 .irq_mask = disable_dynirq, 2127 .irq_unmask = enable_dynirq, 2128 2129 .irq_ack = ack_dynirq, 2130 .irq_mask_ack = mask_ack_dynirq, 2131 2132 .irq_set_affinity = set_affinity_irq, 2133 .irq_retrigger = retrigger_dynirq, 2134}; 2135 2136static struct irq_chip xen_lateeoi_chip __read_mostly = { 2137 /* The chip name needs to contain "xen-dyn" for irqbalance to work. */ 2138 .name = "xen-dyn-lateeoi", 2139 2140 .irq_disable = disable_dynirq, 2141 .irq_mask = disable_dynirq, 2142 .irq_unmask = enable_dynirq, 2143 2144 .irq_ack = lateeoi_ack_dynirq, 2145 .irq_mask_ack = lateeoi_mask_ack_dynirq, 2146 2147 .irq_set_affinity = set_affinity_irq, 2148 .irq_retrigger = retrigger_dynirq, 2149}; 2150 2151static struct irq_chip xen_pirq_chip __read_mostly = { 2152 .name = "xen-pirq", 2153 2154 .irq_startup = startup_pirq, 2155 .irq_shutdown = shutdown_pirq, 2156 .irq_enable = enable_pirq, 2157 .irq_disable = disable_pirq, 2158 2159 .irq_mask = disable_dynirq, 2160 .irq_unmask = enable_dynirq, 2161 2162 .irq_ack = eoi_pirq, 2163 .irq_eoi = eoi_pirq, 2164 .irq_mask_ack = mask_ack_pirq, 2165 2166 .irq_set_affinity = set_affinity_irq, 2167 2168 .irq_retrigger = retrigger_dynirq, 2169}; 2170 2171static struct irq_chip xen_percpu_chip __read_mostly = { 2172 .name = "xen-percpu", 2173 2174 .irq_disable = disable_dynirq, 2175 .irq_mask = disable_dynirq, 2176 .irq_unmask = enable_dynirq, 2177 2178 .irq_ack = ack_dynirq, 2179}; 2180 2181#ifdef CONFIG_X86 2182#ifdef CONFIG_XEN_PVHVM 2183/* Vector callbacks are better than PCI interrupts to receive event 2184 * channel notifications because we can receive vector callbacks on any 2185 * vcpu and we don't need PCI support or APIC interactions. */ 2186void xen_setup_callback_vector(void) 2187{ 2188 uint64_t callback_via; 2189 2190 if (xen_have_vector_callback) { 2191 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR); 2192 if (xen_set_callback_via(callback_via)) { 2193 pr_err("Request for Xen HVM callback vector failed\n"); 2194 xen_have_vector_callback = false; 2195 } 2196 } 2197} 2198 2199/* 2200 * Setup per-vCPU vector-type callbacks. If this setup is unavailable, 2201 * fallback to the global vector-type callback. 2202 */ 2203static __init void xen_init_setup_upcall_vector(void) 2204{ 2205 if (!xen_have_vector_callback) 2206 return; 2207 2208 if ((cpuid_eax(xen_cpuid_base() + 4) & XEN_HVM_CPUID_UPCALL_VECTOR) && 2209 !xen_set_upcall_vector(0)) 2210 xen_percpu_upcall = true; 2211 else if (xen_feature(XENFEAT_hvm_callback_vector)) 2212 xen_setup_callback_vector(); 2213 else 2214 xen_have_vector_callback = false; 2215} 2216 2217int xen_set_upcall_vector(unsigned int cpu) 2218{ 2219 int rc; 2220 xen_hvm_evtchn_upcall_vector_t op = { 2221 .vector = HYPERVISOR_CALLBACK_VECTOR, 2222 .vcpu = per_cpu(xen_vcpu_id, cpu), 2223 }; 2224 2225 rc = HYPERVISOR_hvm_op(HVMOP_set_evtchn_upcall_vector, &op); 2226 if (rc) 2227 return rc; 2228 2229 /* Trick toolstack to think we are enlightened. */ 2230 if (!cpu) 2231 rc = xen_set_callback_via(1); 2232 2233 return rc; 2234} 2235 2236static __init void xen_alloc_callback_vector(void) 2237{ 2238 if (!xen_have_vector_callback) 2239 return; 2240 2241 pr_info("Xen HVM callback vector for event delivery is enabled\n"); 2242 sysvec_install(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback); 2243} 2244#else 2245void xen_setup_callback_vector(void) {} 2246static inline void xen_init_setup_upcall_vector(void) {} 2247int xen_set_upcall_vector(unsigned int cpu) {} 2248static inline void xen_alloc_callback_vector(void) {} 2249#endif /* CONFIG_XEN_PVHVM */ 2250#endif /* CONFIG_X86 */ 2251 2252bool xen_fifo_events = true; 2253module_param_named(fifo_events, xen_fifo_events, bool, 0); 2254 2255static int xen_evtchn_cpu_prepare(unsigned int cpu) 2256{ 2257 int ret = 0; 2258 2259 xen_cpu_init_eoi(cpu); 2260 2261 if (evtchn_ops->percpu_init) 2262 ret = evtchn_ops->percpu_init(cpu); 2263 2264 return ret; 2265} 2266 2267static int xen_evtchn_cpu_dead(unsigned int cpu) 2268{ 2269 int ret = 0; 2270 2271 if (evtchn_ops->percpu_deinit) 2272 ret = evtchn_ops->percpu_deinit(cpu); 2273 2274 return ret; 2275} 2276 2277void __init xen_init_IRQ(void) 2278{ 2279 int ret = -EINVAL; 2280 evtchn_port_t evtchn; 2281 2282 if (xen_fifo_events) 2283 ret = xen_evtchn_fifo_init(); 2284 if (ret < 0) { 2285 xen_evtchn_2l_init(); 2286 xen_fifo_events = false; 2287 } 2288 2289 xen_cpu_init_eoi(smp_processor_id()); 2290 2291 cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE, 2292 "xen/evtchn:prepare", 2293 xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead); 2294 2295 evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()), 2296 sizeof(*evtchn_to_irq), GFP_KERNEL); 2297 BUG_ON(!evtchn_to_irq); 2298 2299 /* No event channels are 'live' right now. */ 2300 for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++) 2301 mask_evtchn(evtchn); 2302 2303 pirq_needs_eoi = pirq_needs_eoi_flag; 2304 2305#ifdef CONFIG_X86 2306 if (xen_pv_domain()) { 2307 if (xen_initial_domain()) 2308 pci_xen_initial_domain(); 2309 } 2310 xen_init_setup_upcall_vector(); 2311 xen_alloc_callback_vector(); 2312 2313 2314 if (xen_hvm_domain()) { 2315 native_init_IRQ(); 2316 /* pci_xen_hvm_init must be called after native_init_IRQ so that 2317 * __acpi_register_gsi can point at the right function */ 2318 pci_xen_hvm_init(); 2319 } else { 2320 int rc; 2321 struct physdev_pirq_eoi_gmfn eoi_gmfn; 2322 2323 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO); 2324 eoi_gmfn.gmfn = virt_to_gfn(pirq_eoi_map); 2325 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn); 2326 if (rc != 0) { 2327 free_page((unsigned long) pirq_eoi_map); 2328 pirq_eoi_map = NULL; 2329 } else 2330 pirq_needs_eoi = pirq_check_eoi_map; 2331 } 2332#endif 2333}
~/
Read-only file preview.