ioapic.c

  1// SPDX-License-Identifier: LGPL-2.1-or-later
  2/*
  3 *  Copyright (C) 2001  MandrakeSoft S.A.
  4 *  Copyright 2010 Red Hat, Inc. and/or its affiliates.
  5 *
  6 *    MandrakeSoft S.A.
  7 *    43, rue d'Aboukir
  8 *    75002 Paris - France
  9 *    http://www.linux-mandrake.com/
 10 *    http://www.mandrakesoft.com/
 11 *
 12 *  Yunhong Jiang <yunhong.jiang@intel.com>
 13 *  Yaozu (Eddie) Dong <eddie.dong@intel.com>
 14 *  Based on Xen 3.1 code.
 15 */
 16#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 17
 18#include <linux/kvm_host.h>
 19#include <linux/kvm.h>
 20#include <linux/mm.h>
 21#include <linux/highmem.h>
 22#include <linux/smp.h>
 23#include <linux/hrtimer.h>
 24#include <linux/io.h>
 25#include <linux/slab.h>
 26#include <linux/export.h>
 27#include <linux/nospec.h>
 28#include <asm/processor.h>
 29#include <asm/page.h>
 30#include <asm/current.h>
 31
 32#include "ioapic.h"
 33#include "lapic.h"
 34#include "irq.h"
 35#include "trace.h"
 36
 37static int ioapic_service(struct kvm_ioapic *vioapic, int irq,
 38		bool line_status);
 39
 40static void kvm_ioapic_update_eoi_one(struct kvm_vcpu *vcpu,
 41				      struct kvm_ioapic *ioapic,
 42				      int trigger_mode,
 43				      int pin);
 44
 45static unsigned long ioapic_read_indirect(struct kvm_ioapic *ioapic)
 46{
 47	unsigned long result = 0;
 48
 49	switch (ioapic->ioregsel) {
 50	case IOAPIC_REG_VERSION:
 51		result = ((((IOAPIC_NUM_PINS - 1) & 0xff) << 16)
 52			  | (IOAPIC_VERSION_ID & 0xff));
 53		break;
 54
 55	case IOAPIC_REG_APIC_ID:
 56	case IOAPIC_REG_ARB_ID:
 57		result = ((ioapic->id & 0xf) << 24);
 58		break;
 59
 60	default:
 61		{
 62			u32 redir_index = (ioapic->ioregsel - 0x10) >> 1;
 63			u64 redir_content = ~0ULL;
 64
 65			if (redir_index < IOAPIC_NUM_PINS) {
 66				u32 index = array_index_nospec(
 67					redir_index, IOAPIC_NUM_PINS);
 68
 69				redir_content = ioapic->redirtbl[index].bits;
 70			}
 71
 72			result = (ioapic->ioregsel & 0x1) ?
 73			    (redir_content >> 32) & 0xffffffff :
 74			    redir_content & 0xffffffff;
 75			break;
 76		}
 77	}
 78
 79	return result;
 80}
 81
 82static void rtc_irq_eoi_tracking_reset(struct kvm_ioapic *ioapic)
 83{
 84	ioapic->rtc_status.pending_eoi = 0;
 85	bitmap_zero(ioapic->rtc_status.dest_map.map, KVM_MAX_VCPU_IDS);
 86}
 87
 88static void kvm_rtc_eoi_tracking_restore_all(struct kvm_ioapic *ioapic);
 89
 90static void rtc_status_pending_eoi_check_valid(struct kvm_ioapic *ioapic)
 91{
 92	if (WARN_ON(ioapic->rtc_status.pending_eoi < 0))
 93		kvm_rtc_eoi_tracking_restore_all(ioapic);
 94}
 95
 96static void __rtc_irq_eoi_tracking_restore_one(struct kvm_vcpu *vcpu)
 97{
 98	bool new_val, old_val;
 99	struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
100	struct dest_map *dest_map = &ioapic->rtc_status.dest_map;
101	union kvm_ioapic_redirect_entry *e;
102
103	e = &ioapic->redirtbl[RTC_GSI];
104	if (!kvm_apic_match_dest(vcpu, NULL, APIC_DEST_NOSHORT,
105				 e->fields.dest_id,
106				 kvm_lapic_irq_dest_mode(!!e->fields.dest_mode)))
107		return;
108
109	new_val = kvm_apic_pending_eoi(vcpu, e->fields.vector);
110	old_val = test_bit(vcpu->vcpu_id, dest_map->map);
111
112	if (new_val == old_val)
113		return;
114
115	if (new_val) {
116		__set_bit(vcpu->vcpu_id, dest_map->map);
117		dest_map->vectors[vcpu->vcpu_id] = e->fields.vector;
118		ioapic->rtc_status.pending_eoi++;
119	} else {
120		__clear_bit(vcpu->vcpu_id, dest_map->map);
121		ioapic->rtc_status.pending_eoi--;
122		rtc_status_pending_eoi_check_valid(ioapic);
123	}
124}
125
126void kvm_rtc_eoi_tracking_restore_one(struct kvm_vcpu *vcpu)
127{
128	struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
129
130	spin_lock(&ioapic->lock);
131	__rtc_irq_eoi_tracking_restore_one(vcpu);
132	spin_unlock(&ioapic->lock);
133}
134
135static void kvm_rtc_eoi_tracking_restore_all(struct kvm_ioapic *ioapic)
136{
137	struct kvm_vcpu *vcpu;
138	unsigned long i;
139
140	if (RTC_GSI >= IOAPIC_NUM_PINS)
141		return;
142
143	rtc_irq_eoi_tracking_reset(ioapic);
144	kvm_for_each_vcpu(i, vcpu, ioapic->kvm)
145	    __rtc_irq_eoi_tracking_restore_one(vcpu);
146}
147
148static void rtc_irq_eoi(struct kvm_ioapic *ioapic, struct kvm_vcpu *vcpu,
149			int vector)
150{
151	struct dest_map *dest_map = &ioapic->rtc_status.dest_map;
152
153	/* RTC special handling */
154	if (test_bit(vcpu->vcpu_id, dest_map->map) &&
155	    (vector == dest_map->vectors[vcpu->vcpu_id]) &&
156	    (test_and_clear_bit(vcpu->vcpu_id,
157				ioapic->rtc_status.dest_map.map))) {
158		--ioapic->rtc_status.pending_eoi;
159		rtc_status_pending_eoi_check_valid(ioapic);
160	}
161}
162
163static bool rtc_irq_check_coalesced(struct kvm_ioapic *ioapic)
164{
165	if (ioapic->rtc_status.pending_eoi > 0)
166		return true; /* coalesced */
167
168	return false;
169}
170
171static void ioapic_lazy_update_eoi(struct kvm_ioapic *ioapic, int irq)
172{
173	unsigned long i;
174	struct kvm_vcpu *vcpu;
175	union kvm_ioapic_redirect_entry *entry = &ioapic->redirtbl[irq];
176
177	kvm_for_each_vcpu(i, vcpu, ioapic->kvm) {
178		if (!kvm_apic_match_dest(vcpu, NULL, APIC_DEST_NOSHORT,
179					 entry->fields.dest_id,
180					 entry->fields.dest_mode) ||
181		    kvm_apic_pending_eoi(vcpu, entry->fields.vector))
182			continue;
183
184		/*
185		 * If no longer has pending EOI in LAPICs, update
186		 * EOI for this vector.
187		 */
188		rtc_irq_eoi(ioapic, vcpu, entry->fields.vector);
189		break;
190	}
191}
192
193static int ioapic_set_irq(struct kvm_ioapic *ioapic, unsigned int irq,
194		int irq_level, bool line_status)
195{
196	union kvm_ioapic_redirect_entry entry;
197	u32 mask = 1 << irq;
198	u32 old_irr;
199	int edge, ret;
200
201	entry = ioapic->redirtbl[irq];
202	edge = (entry.fields.trig_mode == IOAPIC_EDGE_TRIG);
203
204	if (!irq_level) {
205		ioapic->irr &= ~mask;
206		ret = 1;
207		goto out;
208	}
209
210	/*
211	 * AMD SVM AVIC accelerate EOI write iff the interrupt is edge
212	 * triggered, in which case the in-kernel IOAPIC will not be able
213	 * to receive the EOI.  In this case, we do a lazy update of the
214	 * pending EOI when trying to set IOAPIC irq.
215	 */
216	if (edge && kvm_apicv_activated(ioapic->kvm))
217		ioapic_lazy_update_eoi(ioapic, irq);
218
219	/*
220	 * Return 0 for coalesced interrupts; for edge-triggered interrupts,
221	 * this only happens if a previous edge has not been delivered due
222	 * to masking.  For level interrupts, the remote_irr field tells
223	 * us if the interrupt is waiting for an EOI.
224	 *
225	 * RTC is special: it is edge-triggered, but userspace likes to know
226	 * if it has been already ack-ed via EOI because coalesced RTC
227	 * interrupts lead to time drift in Windows guests.  So we track
228	 * EOI manually for the RTC interrupt.
229	 */
230	if (irq == RTC_GSI && line_status &&
231		rtc_irq_check_coalesced(ioapic)) {
232		ret = 0;
233		goto out;
234	}
235
236	old_irr = ioapic->irr;
237	ioapic->irr |= mask;
238	if (edge) {
239		ioapic->irr_delivered &= ~mask;
240		if (old_irr == ioapic->irr) {
241			ret = 0;
242			goto out;
243		}
244	}
245
246	ret = ioapic_service(ioapic, irq, line_status);
247
248out:
249	trace_kvm_ioapic_set_irq(entry.bits, irq, ret == 0);
250	return ret;
251}
252
253static void kvm_ioapic_inject_all(struct kvm_ioapic *ioapic, unsigned long irr)
254{
255	u32 idx;
256
257	rtc_irq_eoi_tracking_reset(ioapic);
258	for_each_set_bit(idx, &irr, IOAPIC_NUM_PINS)
259		ioapic_set_irq(ioapic, idx, 1, true);
260
261	kvm_rtc_eoi_tracking_restore_all(ioapic);
262}
263
264
265void kvm_ioapic_scan_entry(struct kvm_vcpu *vcpu, ulong *ioapic_handled_vectors)
266{
267	struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
268	struct dest_map *dest_map = &ioapic->rtc_status.dest_map;
269	union kvm_ioapic_redirect_entry *e;
270	int index;
271
272	spin_lock(&ioapic->lock);
273
274	/* Make sure we see any missing RTC EOI */
275	if (test_bit(vcpu->vcpu_id, dest_map->map))
276		__set_bit(dest_map->vectors[vcpu->vcpu_id],
277			  ioapic_handled_vectors);
278
279	for (index = 0; index < IOAPIC_NUM_PINS; index++) {
280		e = &ioapic->redirtbl[index];
281		if (e->fields.trig_mode == IOAPIC_LEVEL_TRIG ||
282		    kvm_irq_has_notifier(ioapic->kvm, KVM_IRQCHIP_IOAPIC, index) ||
283		    index == RTC_GSI) {
284			u16 dm = kvm_lapic_irq_dest_mode(!!e->fields.dest_mode);
285
286			kvm_scan_ioapic_irq(vcpu, e->fields.dest_id, dm,
287					    e->fields.vector, ioapic_handled_vectors);
288		}
289	}
290	spin_unlock(&ioapic->lock);
291}
292
293void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm)
294{
295	if (!ioapic_in_kernel(kvm))
296		return;
297	kvm_make_scan_ioapic_request(kvm);
298}
299
300void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
301				    struct kvm_irq_mask_notifier *kimn)
302{
303	struct kvm_ioapic *ioapic = kvm->arch.vioapic;
304
305	mutex_lock(&kvm->irq_lock);
306	kimn->irq = irq;
307	hlist_add_head_rcu(&kimn->link, &ioapic->mask_notifier_list);
308	mutex_unlock(&kvm->irq_lock);
309}
310
311void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
312				      struct kvm_irq_mask_notifier *kimn)
313{
314	mutex_lock(&kvm->irq_lock);
315	hlist_del_rcu(&kimn->link);
316	mutex_unlock(&kvm->irq_lock);
317	synchronize_srcu(&kvm->irq_srcu);
318}
319
320void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
321			     bool mask)
322{
323	struct kvm_ioapic *ioapic = kvm->arch.vioapic;
324	struct kvm_irq_mask_notifier *kimn;
325	int idx, gsi;
326
327	idx = srcu_read_lock(&kvm->irq_srcu);
328	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
329	if (gsi != -1)
330		hlist_for_each_entry_rcu(kimn, &ioapic->mask_notifier_list, link)
331			if (kimn->irq == gsi)
332				kimn->func(kimn, mask);
333	srcu_read_unlock(&kvm->irq_srcu, idx);
334}
335
336static void ioapic_write_indirect(struct kvm_ioapic *ioapic, u32 val)
337{
338	unsigned index;
339	bool mask_before, mask_after;
340	union kvm_ioapic_redirect_entry *e;
341	int old_remote_irr, old_delivery_status, old_dest_id, old_dest_mode;
342	DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS);
343
344	switch (ioapic->ioregsel) {
345	case IOAPIC_REG_VERSION:
346		/* Writes are ignored. */
347		break;
348
349	case IOAPIC_REG_APIC_ID:
350		ioapic->id = (val >> 24) & 0xf;
351		break;
352
353	case IOAPIC_REG_ARB_ID:
354		break;
355
356	default:
357		index = (ioapic->ioregsel - 0x10) >> 1;
358
359		if (index >= IOAPIC_NUM_PINS)
360			return;
361		index = array_index_nospec(index, IOAPIC_NUM_PINS);
362		e = &ioapic->redirtbl[index];
363		mask_before = e->fields.mask;
364		/* Preserve read-only fields */
365		old_remote_irr = e->fields.remote_irr;
366		old_delivery_status = e->fields.delivery_status;
367		old_dest_id = e->fields.dest_id;
368		old_dest_mode = e->fields.dest_mode;
369		if (ioapic->ioregsel & 1) {
370			e->bits &= 0xffffffff;
371			e->bits |= (u64) val << 32;
372		} else {
373			e->bits &= ~0xffffffffULL;
374			e->bits |= (u32) val;
375		}
376		e->fields.remote_irr = old_remote_irr;
377		e->fields.delivery_status = old_delivery_status;
378
379		/*
380		 * Some OSes (Linux, Xen) assume that Remote IRR bit will
381		 * be cleared by IOAPIC hardware when the entry is configured
382		 * as edge-triggered. This behavior is used to simulate an
383		 * explicit EOI on IOAPICs that don't have the EOI register.
384		 */
385		if (e->fields.trig_mode == IOAPIC_EDGE_TRIG)
386			e->fields.remote_irr = 0;
387
388		mask_after = e->fields.mask;
389		if (mask_before != mask_after)
390			kvm_fire_mask_notifiers(ioapic->kvm, KVM_IRQCHIP_IOAPIC, index, mask_after);
391		if (e->fields.trig_mode == IOAPIC_LEVEL_TRIG &&
392		    ioapic->irr & (1 << index) && !e->fields.mask && !e->fields.remote_irr) {
393			/*
394			 * Pending status in irr may be outdated: the IRQ line may have
395			 * already been deasserted by a device while the IRQ was masked.
396			 * This occurs, for instance, if the interrupt is handled in a
397			 * Linux guest as a oneshot interrupt (IRQF_ONESHOT). In this
398			 * case the guest acknowledges the interrupt to the device in
399			 * its threaded irq handler, i.e. after the EOI but before
400			 * unmasking, so at the time of unmasking the IRQ line is
401			 * already down but our pending irr bit is still set. In such
402			 * cases, injecting this pending interrupt to the guest is
403			 * buggy: the guest will receive an extra unwanted interrupt.
404			 *
405			 * So we need to check here if the IRQ is actually still pending.
406			 * As we are generally not able to probe the IRQ line status
407			 * directly, we do it through irqfd resampler. Namely, we clear
408			 * the pending status and notify the resampler that this interrupt
409			 * is done, without actually injecting it into the guest. If the
410			 * IRQ line is actually already deasserted, we are done. If it is
411			 * still asserted, a new interrupt will be shortly triggered
412			 * through irqfd and injected into the guest.
413			 *
414			 * If, however, it's not possible to resample (no irqfd resampler
415			 * registered for this irq), then unconditionally inject this
416			 * pending interrupt into the guest, so the guest will not miss
417			 * an interrupt, although may get an extra unwanted interrupt.
418			 */
419			if (kvm_notify_irqfd_resampler(ioapic->kvm, KVM_IRQCHIP_IOAPIC, index))
420				ioapic->irr &= ~(1 << index);
421			else
422				ioapic_service(ioapic, index, false);
423		}
424		if (e->fields.delivery_mode == APIC_DM_FIXED) {
425			struct kvm_lapic_irq irq;
426
427			irq.vector = e->fields.vector;
428			irq.delivery_mode = e->fields.delivery_mode << 8;
429			irq.dest_mode =
430			    kvm_lapic_irq_dest_mode(!!e->fields.dest_mode);
431			irq.level = false;
432			irq.trig_mode = e->fields.trig_mode;
433			irq.shorthand = APIC_DEST_NOSHORT;
434			irq.dest_id = e->fields.dest_id;
435			irq.msi_redir_hint = false;
436			bitmap_zero(vcpu_bitmap, KVM_MAX_VCPUS);
437			kvm_bitmap_or_dest_vcpus(ioapic->kvm, &irq,
438						 vcpu_bitmap);
439			if (old_dest_mode != e->fields.dest_mode ||
440			    old_dest_id != e->fields.dest_id) {
441				/*
442				 * Update vcpu_bitmap with vcpus specified in
443				 * the previous request as well. This is done to
444				 * keep ioapic_handled_vectors synchronized.
445				 */
446				irq.dest_id = old_dest_id;
447				irq.dest_mode =
448				    kvm_lapic_irq_dest_mode(
449					!!e->fields.dest_mode);
450				kvm_bitmap_or_dest_vcpus(ioapic->kvm, &irq,
451							 vcpu_bitmap);
452			}
453			kvm_make_scan_ioapic_request_mask(ioapic->kvm,
454							  vcpu_bitmap);
455		} else {
456			kvm_make_scan_ioapic_request(ioapic->kvm);
457		}
458		break;
459	}
460}
461
462static int ioapic_service(struct kvm_ioapic *ioapic, int irq, bool line_status)
463{
464	union kvm_ioapic_redirect_entry *entry = &ioapic->redirtbl[irq];
465	struct kvm_lapic_irq irqe;
466	int ret;
467
468	if (entry->fields.mask ||
469	    (entry->fields.trig_mode == IOAPIC_LEVEL_TRIG &&
470	    entry->fields.remote_irr))
471		return -1;
472
473	irqe.dest_id = entry->fields.dest_id;
474	irqe.vector = entry->fields.vector;
475	irqe.dest_mode = kvm_lapic_irq_dest_mode(!!entry->fields.dest_mode);
476	irqe.trig_mode = entry->fields.trig_mode;
477	irqe.delivery_mode = entry->fields.delivery_mode << 8;
478	irqe.level = 1;
479	irqe.shorthand = APIC_DEST_NOSHORT;
480	irqe.msi_redir_hint = false;
481
482	if (irqe.trig_mode == IOAPIC_EDGE_TRIG)
483		ioapic->irr_delivered |= 1 << irq;
484
485	if (irq == RTC_GSI && line_status) {
486		/*
487		 * pending_eoi cannot ever become negative (see
488		 * rtc_status_pending_eoi_check_valid) and the caller
489		 * ensures that it is only called if it is >= zero, namely
490		 * if rtc_irq_check_coalesced returns false).
491		 */
492		BUG_ON(ioapic->rtc_status.pending_eoi != 0);
493		ret = kvm_irq_delivery_to_apic(ioapic->kvm, NULL, &irqe,
494					       &ioapic->rtc_status.dest_map);
495		ioapic->rtc_status.pending_eoi = (ret < 0 ? 0 : ret);
496	} else
497		ret = kvm_irq_delivery_to_apic(ioapic->kvm, NULL, &irqe, NULL);
498
499	if (ret && irqe.trig_mode == IOAPIC_LEVEL_TRIG)
500		entry->fields.remote_irr = 1;
501
502	return ret;
503}
504
505int kvm_ioapic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
506		       int irq_source_id, int level, bool line_status)
507{
508	struct kvm_ioapic *ioapic = kvm->arch.vioapic;
509	int irq = e->irqchip.pin;
510	int ret, irq_level;
511
512	BUG_ON(irq < 0 || irq >= IOAPIC_NUM_PINS);
513
514	spin_lock(&ioapic->lock);
515	irq_level = __kvm_irq_line_state(&ioapic->irq_states[irq],
516					 irq_source_id, level);
517	ret = ioapic_set_irq(ioapic, irq, irq_level, line_status);
518
519	spin_unlock(&ioapic->lock);
520
521	return ret;
522}
523
524static void kvm_ioapic_eoi_inject_work(struct work_struct *work)
525{
526	int i;
527	struct kvm_ioapic *ioapic = container_of(work, struct kvm_ioapic,
528						 eoi_inject.work);
529	spin_lock(&ioapic->lock);
530	for (i = 0; i < IOAPIC_NUM_PINS; i++) {
531		union kvm_ioapic_redirect_entry *ent = &ioapic->redirtbl[i];
532
533		if (ent->fields.trig_mode != IOAPIC_LEVEL_TRIG)
534			continue;
535
536		if (ioapic->irr & (1 << i) && !ent->fields.remote_irr)
537			ioapic_service(ioapic, i, false);
538	}
539	spin_unlock(&ioapic->lock);
540}
541
542#define IOAPIC_SUCCESSIVE_IRQ_MAX_COUNT 10000
543static void kvm_ioapic_update_eoi_one(struct kvm_vcpu *vcpu,
544				      struct kvm_ioapic *ioapic,
545				      int trigger_mode,
546				      int pin)
547{
548	struct kvm_lapic *apic = vcpu->arch.apic;
549	union kvm_ioapic_redirect_entry *ent = &ioapic->redirtbl[pin];
550
551	/*
552	 * We are dropping lock while calling ack notifiers because ack
553	 * notifier callbacks for assigned devices call into IOAPIC
554	 * recursively. Since remote_irr is cleared only after call
555	 * to notifiers if the same vector will be delivered while lock
556	 * is dropped it will be put into irr and will be delivered
557	 * after ack notifier returns.
558	 */
559	spin_unlock(&ioapic->lock);
560	kvm_notify_acked_irq(ioapic->kvm, KVM_IRQCHIP_IOAPIC, pin);
561	spin_lock(&ioapic->lock);
562
563	if (trigger_mode != IOAPIC_LEVEL_TRIG ||
564	    kvm_lapic_suppress_eoi_broadcast(apic))
565		return;
566
567	ASSERT(ent->fields.trig_mode == IOAPIC_LEVEL_TRIG);
568	ent->fields.remote_irr = 0;
569	if (!ent->fields.mask && (ioapic->irr & (1 << pin))) {
570		++ioapic->irq_eoi[pin];
571		if (ioapic->irq_eoi[pin] == IOAPIC_SUCCESSIVE_IRQ_MAX_COUNT) {
572			/*
573			 * Real hardware does not deliver the interrupt
574			 * immediately during eoi broadcast, and this
575			 * lets a buggy guest make slow progress
576			 * even if it does not correctly handle a
577			 * level-triggered interrupt.  Emulate this
578			 * behavior if we detect an interrupt storm.
579			 */
580			schedule_delayed_work(&ioapic->eoi_inject, HZ / 100);
581			ioapic->irq_eoi[pin] = 0;
582			trace_kvm_ioapic_delayed_eoi_inj(ent->bits);
583		} else {
584			ioapic_service(ioapic, pin, false);
585		}
586	} else {
587		ioapic->irq_eoi[pin] = 0;
588	}
589}
590
591void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector, int trigger_mode)
592{
593	int i;
594	struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
595
596	spin_lock(&ioapic->lock);
597	rtc_irq_eoi(ioapic, vcpu, vector);
598	for (i = 0; i < IOAPIC_NUM_PINS; i++) {
599		union kvm_ioapic_redirect_entry *ent = &ioapic->redirtbl[i];
600
601		if (ent->fields.vector != vector)
602			continue;
603		kvm_ioapic_update_eoi_one(vcpu, ioapic, trigger_mode, i);
604	}
605	spin_unlock(&ioapic->lock);
606}
607
608static inline struct kvm_ioapic *to_ioapic(struct kvm_io_device *dev)
609{
610	return container_of(dev, struct kvm_ioapic, dev);
611}
612
613static inline int ioapic_in_range(struct kvm_ioapic *ioapic, gpa_t addr)
614{
615	return ((addr >= ioapic->base_address &&
616		 (addr < ioapic->base_address + IOAPIC_MEM_LENGTH)));
617}
618
619static int ioapic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
620				gpa_t addr, int len, void *val)
621{
622	struct kvm_ioapic *ioapic = to_ioapic(this);
623	u32 result;
624	if (!ioapic_in_range(ioapic, addr))
625		return -EOPNOTSUPP;
626
627	ASSERT(!(addr & 0xf));	/* check alignment */
628
629	addr &= 0xff;
630	spin_lock(&ioapic->lock);
631	switch (addr) {
632	case IOAPIC_REG_SELECT:
633		result = ioapic->ioregsel;
634		break;
635
636	case IOAPIC_REG_WINDOW:
637		result = ioapic_read_indirect(ioapic);
638		break;
639
640	default:
641		result = 0;
642		break;
643	}
644	spin_unlock(&ioapic->lock);
645
646	switch (len) {
647	case 8:
648		*(u64 *) val = result;
649		break;
650	case 1:
651	case 2:
652	case 4:
653		memcpy(val, (char *)&result, len);
654		break;
655	default:
656		printk(KERN_WARNING "ioapic: wrong length %d\n", len);
657	}
658	return 0;
659}
660
661static int ioapic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
662				 gpa_t addr, int len, const void *val)
663{
664	struct kvm_ioapic *ioapic = to_ioapic(this);
665	u32 data;
666	if (!ioapic_in_range(ioapic, addr))
667		return -EOPNOTSUPP;
668
669	ASSERT(!(addr & 0xf));	/* check alignment */
670
671	switch (len) {
672	case 8:
673	case 4:
674		data = *(u32 *) val;
675		break;
676	case 2:
677		data = *(u16 *) val;
678		break;
679	case 1:
680		data = *(u8  *) val;
681		break;
682	default:
683		printk(KERN_WARNING "ioapic: Unsupported size %d\n", len);
684		return 0;
685	}
686
687	addr &= 0xff;
688	spin_lock(&ioapic->lock);
689	switch (addr) {
690	case IOAPIC_REG_SELECT:
691		ioapic->ioregsel = data & 0xFF; /* 8-bit register */
692		break;
693
694	case IOAPIC_REG_WINDOW:
695		ioapic_write_indirect(ioapic, data);
696		break;
697
698	default:
699		break;
700	}
701	spin_unlock(&ioapic->lock);
702	return 0;
703}
704
705static void kvm_ioapic_reset(struct kvm_ioapic *ioapic)
706{
707	int i;
708
709	cancel_delayed_work_sync(&ioapic->eoi_inject);
710	for (i = 0; i < IOAPIC_NUM_PINS; i++)
711		ioapic->redirtbl[i].fields.mask = 1;
712	ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS;
713	ioapic->ioregsel = 0;
714	ioapic->irr = 0;
715	ioapic->irr_delivered = 0;
716	ioapic->id = 0;
717	memset(ioapic->irq_eoi, 0x00, sizeof(ioapic->irq_eoi));
718	rtc_irq_eoi_tracking_reset(ioapic);
719}
720
721static const struct kvm_io_device_ops ioapic_mmio_ops = {
722	.read     = ioapic_mmio_read,
723	.write    = ioapic_mmio_write,
724};
725
726int kvm_ioapic_init(struct kvm *kvm)
727{
728	struct kvm_ioapic *ioapic;
729	int ret;
730
731	ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL_ACCOUNT);
732	if (!ioapic)
733		return -ENOMEM;
734	spin_lock_init(&ioapic->lock);
735	INIT_DELAYED_WORK(&ioapic->eoi_inject, kvm_ioapic_eoi_inject_work);
736	INIT_HLIST_HEAD(&ioapic->mask_notifier_list);
737	kvm->arch.vioapic = ioapic;
738	kvm_ioapic_reset(ioapic);
739	kvm_iodevice_init(&ioapic->dev, &ioapic_mmio_ops);
740	ioapic->kvm = kvm;
741	mutex_lock(&kvm->slots_lock);
742	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, ioapic->base_address,
743				      IOAPIC_MEM_LENGTH, &ioapic->dev);
744	mutex_unlock(&kvm->slots_lock);
745	if (ret < 0) {
746		kvm->arch.vioapic = NULL;
747		kfree(ioapic);
748	}
749
750	return ret;
751}
752
753void kvm_ioapic_destroy(struct kvm *kvm)
754{
755	struct kvm_ioapic *ioapic = kvm->arch.vioapic;
756
757	if (!ioapic)
758		return;
759
760	cancel_delayed_work_sync(&ioapic->eoi_inject);
761	mutex_lock(&kvm->slots_lock);
762	kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &ioapic->dev);
763	mutex_unlock(&kvm->slots_lock);
764	kvm->arch.vioapic = NULL;
765	kfree(ioapic);
766}
767
768void kvm_get_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state)
769{
770	struct kvm_ioapic *ioapic = kvm->arch.vioapic;
771
772	spin_lock(&ioapic->lock);
773	memcpy(state, ioapic, sizeof(struct kvm_ioapic_state));
774	state->irr &= ~ioapic->irr_delivered;
775	spin_unlock(&ioapic->lock);
776}
777
778void kvm_set_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state)
779{
780	struct kvm_ioapic *ioapic = kvm->arch.vioapic;
781
782	spin_lock(&ioapic->lock);
783	memcpy(ioapic, state, sizeof(struct kvm_ioapic_state));
784	ioapic->irr = 0;
785	ioapic->irr_delivered = 0;
786	kvm_make_scan_ioapic_request(kvm);
787	kvm_ioapic_inject_all(ioapic, state->irr);
788	spin_unlock(&ioapic->lock);
789}