aia.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
  4 * Copyright (C) 2022 Ventana Micro Systems Inc.
  5 *
  6 * Authors:
  7 *	Anup Patel <apatel@ventanamicro.com>
  8 */
  9
 10#include <linux/kernel.h>
 11#include <linux/bitops.h>
 12#include <linux/irq.h>
 13#include <linux/irqchip/riscv-imsic.h>
 14#include <linux/irqdomain.h>
 15#include <linux/kvm_host.h>
 16#include <linux/percpu.h>
 17#include <linux/spinlock.h>
 18#include <asm/cpufeature.h>
 19#include <asm/kvm_nacl.h>
 20
 21struct aia_hgei_control {
 22	raw_spinlock_t lock;
 23	unsigned long free_bitmap;
 24	struct kvm_vcpu *owners[BITS_PER_LONG];
 25};
 26static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei);
 27static int hgei_parent_irq;
 28
 29unsigned int kvm_riscv_aia_nr_hgei;
 30unsigned int kvm_riscv_aia_max_ids;
 31DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available);
 32
 33static inline unsigned long aia_hvictl_value(bool ext_irq_pending)
 34{
 35	unsigned long hvictl;
 36
 37	/*
 38	 * HVICTL.IID == 9 and HVICTL.IPRIO == 0 represents
 39	 * no interrupt in HVICTL.
 40	 */
 41
 42	hvictl = (IRQ_S_EXT << HVICTL_IID_SHIFT) & HVICTL_IID;
 43	hvictl |= ext_irq_pending;
 44	return hvictl;
 45}
 46
 47#ifdef CONFIG_32BIT
 48void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu)
 49{
 50	struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
 51	unsigned long mask, val;
 52
 53	if (!kvm_riscv_aia_available())
 54		return;
 55
 56	if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) {
 57		mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0);
 58		val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask;
 59
 60		csr->hviph &= ~mask;
 61		csr->hviph |= val;
 62	}
 63}
 64
 65void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu)
 66{
 67	struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
 68
 69	if (kvm_riscv_aia_available())
 70		csr->vsieh = ncsr_read(CSR_VSIEH);
 71}
 72#endif
 73
 74bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
 75{
 76	unsigned long seip;
 77
 78	if (!kvm_riscv_aia_available())
 79		return false;
 80
 81#ifdef CONFIG_32BIT
 82	if (READ_ONCE(vcpu->arch.irqs_pending[1]) &
 83	    (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask)))
 84		return true;
 85#endif
 86
 87	seip = vcpu->arch.guest_csr.vsie;
 88	seip &= (unsigned long)mask;
 89	seip &= BIT(IRQ_S_EXT);
 90
 91	if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip)
 92		return false;
 93
 94	return kvm_riscv_vcpu_aia_imsic_has_interrupt(vcpu);
 95}
 96
 97void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu)
 98{
 99	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
100
101	if (!kvm_riscv_aia_available())
102		return;
103
104#ifdef CONFIG_32BIT
105	ncsr_write(CSR_HVIPH, vcpu->arch.aia_context.guest_csr.hviph);
106#endif
107	ncsr_write(CSR_HVICTL, aia_hvictl_value(!!(csr->hvip & BIT(IRQ_VS_EXT))));
108}
109
110void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu)
111{
112	struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
113	void *nsh;
114
115	if (!kvm_riscv_aia_available())
116		return;
117
118	if (kvm_riscv_nacl_sync_csr_available()) {
119		nsh = nacl_shmem();
120		nacl_csr_write(nsh, CSR_VSISELECT, csr->vsiselect);
121		nacl_csr_write(nsh, CSR_HVIPRIO1, csr->hviprio1);
122		nacl_csr_write(nsh, CSR_HVIPRIO2, csr->hviprio2);
123#ifdef CONFIG_32BIT
124		nacl_csr_write(nsh, CSR_VSIEH, csr->vsieh);
125		nacl_csr_write(nsh, CSR_HVIPH, csr->hviph);
126		nacl_csr_write(nsh, CSR_HVIPRIO1H, csr->hviprio1h);
127		nacl_csr_write(nsh, CSR_HVIPRIO2H, csr->hviprio2h);
128#endif
129	} else {
130		csr_write(CSR_VSISELECT, csr->vsiselect);
131		csr_write(CSR_HVIPRIO1, csr->hviprio1);
132		csr_write(CSR_HVIPRIO2, csr->hviprio2);
133#ifdef CONFIG_32BIT
134		csr_write(CSR_VSIEH, csr->vsieh);
135		csr_write(CSR_HVIPH, csr->hviph);
136		csr_write(CSR_HVIPRIO1H, csr->hviprio1h);
137		csr_write(CSR_HVIPRIO2H, csr->hviprio2h);
138#endif
139	}
140
141	if (kvm_riscv_aia_initialized(vcpu->kvm))
142		kvm_riscv_vcpu_aia_imsic_load(vcpu, cpu);
143}
144
145void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu)
146{
147	struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
148	void *nsh;
149
150	if (!kvm_riscv_aia_available())
151		return;
152
153	if (kvm_riscv_aia_initialized(vcpu->kvm))
154		kvm_riscv_vcpu_aia_imsic_put(vcpu);
155
156	if (kvm_riscv_nacl_available()) {
157		nsh = nacl_shmem();
158		csr->vsiselect = nacl_csr_read(nsh, CSR_VSISELECT);
159		csr->hviprio1 = nacl_csr_read(nsh, CSR_HVIPRIO1);
160		csr->hviprio2 = nacl_csr_read(nsh, CSR_HVIPRIO2);
161#ifdef CONFIG_32BIT
162		csr->vsieh = nacl_csr_read(nsh, CSR_VSIEH);
163		csr->hviph = nacl_csr_read(nsh, CSR_HVIPH);
164		csr->hviprio1h = nacl_csr_read(nsh, CSR_HVIPRIO1H);
165		csr->hviprio2h = nacl_csr_read(nsh, CSR_HVIPRIO2H);
166#endif
167	} else {
168		csr->vsiselect = csr_read(CSR_VSISELECT);
169		csr->hviprio1 = csr_read(CSR_HVIPRIO1);
170		csr->hviprio2 = csr_read(CSR_HVIPRIO2);
171#ifdef CONFIG_32BIT
172		csr->vsieh = csr_read(CSR_VSIEH);
173		csr->hviph = csr_read(CSR_HVIPH);
174		csr->hviprio1h = csr_read(CSR_HVIPRIO1H);
175		csr->hviprio2h = csr_read(CSR_HVIPRIO2H);
176#endif
177	}
178}
179
180int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu,
181			       unsigned long reg_num,
182			       unsigned long *out_val)
183{
184	struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
185
186	if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long))
187		return -ENOENT;
188
189	*out_val = 0;
190	if (kvm_riscv_aia_available())
191		*out_val = ((unsigned long *)csr)[reg_num];
192
193	return 0;
194}
195
196int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu,
197			       unsigned long reg_num,
198			       unsigned long val)
199{
200	struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
201
202	if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long))
203		return -ENOENT;
204
205	if (kvm_riscv_aia_available()) {
206		((unsigned long *)csr)[reg_num] = val;
207
208#ifdef CONFIG_32BIT
209		if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph))
210			WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0);
211#endif
212	}
213
214	return 0;
215}
216
217int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu,
218				 unsigned int csr_num,
219				 unsigned long *val,
220				 unsigned long new_val,
221				 unsigned long wr_mask)
222{
223	/* If AIA not available then redirect trap */
224	if (!kvm_riscv_aia_available())
225		return KVM_INSN_ILLEGAL_TRAP;
226
227	/* If AIA not initialized then forward to user space */
228	if (!kvm_riscv_aia_initialized(vcpu->kvm))
229		return KVM_INSN_EXIT_TO_USER_SPACE;
230
231	return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI,
232					    val, new_val, wr_mask);
233}
234
235/*
236 * External IRQ priority always read-only zero. This means default
237 * priority order  is always preferred for external IRQs unless
238 * HVICTL.IID == 9 and HVICTL.IPRIO != 0
239 */
240static int aia_irq2bitpos[] = {
2410,     8,   -1,   -1,   16,   24,   -1,   -1, /* 0 - 7 */
24232,   -1,   -1,   -1,   -1,   40,   48,   56, /* 8 - 15 */
24364,   72,   80,   88,   96,  104,  112,  120, /* 16 - 23 */
244-1,   -1,   -1,   -1,   -1,   -1,   -1,   -1, /* 24 - 31 */
245-1,   -1,   -1,   -1,   -1,   -1,   -1,   -1, /* 32 - 39 */
246-1,   -1,   -1,   -1,   -1,   -1,   -1,   -1, /* 40 - 47 */
247-1,   -1,   -1,   -1,   -1,   -1,   -1,   -1, /* 48 - 55 */
248-1,   -1,   -1,   -1,   -1,   -1,   -1,   -1, /* 56 - 63 */
249};
250
251static u8 aia_get_iprio8(struct kvm_vcpu *vcpu, unsigned int irq)
252{
253	unsigned long hviprio;
254	int bitpos = aia_irq2bitpos[irq];
255
256	if (bitpos < 0)
257		return 0;
258
259	switch (bitpos / BITS_PER_LONG) {
260	case 0:
261		hviprio = ncsr_read(CSR_HVIPRIO1);
262		break;
263	case 1:
264#ifndef CONFIG_32BIT
265		hviprio = ncsr_read(CSR_HVIPRIO2);
266		break;
267#else
268		hviprio = ncsr_read(CSR_HVIPRIO1H);
269		break;
270	case 2:
271		hviprio = ncsr_read(CSR_HVIPRIO2);
272		break;
273	case 3:
274		hviprio = ncsr_read(CSR_HVIPRIO2H);
275		break;
276#endif
277	default:
278		return 0;
279	}
280
281	return (hviprio >> (bitpos % BITS_PER_LONG)) & TOPI_IPRIO_MASK;
282}
283
284static void aia_set_iprio8(struct kvm_vcpu *vcpu, unsigned int irq, u8 prio)
285{
286	unsigned long hviprio;
287	int bitpos = aia_irq2bitpos[irq];
288
289	if (bitpos < 0)
290		return;
291
292	switch (bitpos / BITS_PER_LONG) {
293	case 0:
294		hviprio = ncsr_read(CSR_HVIPRIO1);
295		break;
296	case 1:
297#ifndef CONFIG_32BIT
298		hviprio = ncsr_read(CSR_HVIPRIO2);
299		break;
300#else
301		hviprio = ncsr_read(CSR_HVIPRIO1H);
302		break;
303	case 2:
304		hviprio = ncsr_read(CSR_HVIPRIO2);
305		break;
306	case 3:
307		hviprio = ncsr_read(CSR_HVIPRIO2H);
308		break;
309#endif
310	default:
311		return;
312	}
313
314	hviprio &= ~(TOPI_IPRIO_MASK << (bitpos % BITS_PER_LONG));
315	hviprio |= (unsigned long)prio << (bitpos % BITS_PER_LONG);
316
317	switch (bitpos / BITS_PER_LONG) {
318	case 0:
319		ncsr_write(CSR_HVIPRIO1, hviprio);
320		break;
321	case 1:
322#ifndef CONFIG_32BIT
323		ncsr_write(CSR_HVIPRIO2, hviprio);
324		break;
325#else
326		ncsr_write(CSR_HVIPRIO1H, hviprio);
327		break;
328	case 2:
329		ncsr_write(CSR_HVIPRIO2, hviprio);
330		break;
331	case 3:
332		ncsr_write(CSR_HVIPRIO2H, hviprio);
333		break;
334#endif
335	default:
336		return;
337	}
338}
339
340static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel,
341			 unsigned long *val, unsigned long new_val,
342			 unsigned long wr_mask)
343{
344	int i, first_irq, nirqs;
345	unsigned long old_val;
346	u8 prio;
347
348#ifndef CONFIG_32BIT
349	if (isel & 0x1)
350		return KVM_INSN_ILLEGAL_TRAP;
351#endif
352
353	nirqs = 4 * (BITS_PER_LONG / 32);
354	first_irq = (isel - ISELECT_IPRIO0) * 4;
355
356	old_val = 0;
357	for (i = 0; i < nirqs; i++) {
358		prio = aia_get_iprio8(vcpu, first_irq + i);
359		old_val |= (unsigned long)prio << (TOPI_IPRIO_BITS * i);
360	}
361
362	if (val)
363		*val = old_val;
364
365	if (wr_mask) {
366		new_val = (old_val & ~wr_mask) | (new_val & wr_mask);
367		for (i = 0; i < nirqs; i++) {
368			prio = (new_val >> (TOPI_IPRIO_BITS * i)) &
369				TOPI_IPRIO_MASK;
370			aia_set_iprio8(vcpu, first_irq + i, prio);
371		}
372	}
373
374	return KVM_INSN_CONTINUE_NEXT_SEPC;
375}
376
377int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num,
378				unsigned long *val, unsigned long new_val,
379				unsigned long wr_mask)
380{
381	unsigned int isel;
382
383	/* If AIA not available then redirect trap */
384	if (!kvm_riscv_aia_available())
385		return KVM_INSN_ILLEGAL_TRAP;
386
387	/* First try to emulate in kernel space */
388	isel = ncsr_read(CSR_VSISELECT) & ISELECT_MASK;
389	if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15)
390		return aia_rmw_iprio(vcpu, isel, val, new_val, wr_mask);
391	else if (isel >= IMSIC_FIRST && isel <= IMSIC_LAST &&
392		 kvm_riscv_aia_initialized(vcpu->kvm))
393		return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, isel, val, new_val,
394						    wr_mask);
395
396	/* We can't handle it here so redirect to user space */
397	return KVM_INSN_EXIT_TO_USER_SPACE;
398}
399
400int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner,
401			     void __iomem **hgei_va, phys_addr_t *hgei_pa)
402{
403	int ret = -ENOENT;
404	unsigned long flags;
405	const struct imsic_global_config *gc;
406	const struct imsic_local_config *lc;
407	struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
408
409	if (!kvm_riscv_aia_available() || !hgctrl)
410		return -ENODEV;
411
412	raw_spin_lock_irqsave(&hgctrl->lock, flags);
413
414	if (hgctrl->free_bitmap) {
415		ret = __ffs(hgctrl->free_bitmap);
416		hgctrl->free_bitmap &= ~BIT(ret);
417		hgctrl->owners[ret] = owner;
418	}
419
420	raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
421
422	gc = imsic_get_global_config();
423	lc = (gc) ? per_cpu_ptr(gc->local, cpu) : NULL;
424	if (lc && ret > 0) {
425		if (hgei_va)
426			*hgei_va = lc->msi_va + (ret * IMSIC_MMIO_PAGE_SZ);
427		if (hgei_pa)
428			*hgei_pa = lc->msi_pa + (ret * IMSIC_MMIO_PAGE_SZ);
429	}
430
431	return ret;
432}
433
434void kvm_riscv_aia_free_hgei(int cpu, int hgei)
435{
436	unsigned long flags;
437	struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
438
439	if (!kvm_riscv_aia_available() || !hgctrl)
440		return;
441
442	raw_spin_lock_irqsave(&hgctrl->lock, flags);
443
444	if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) {
445		if (!(hgctrl->free_bitmap & BIT(hgei))) {
446			hgctrl->free_bitmap |= BIT(hgei);
447			hgctrl->owners[hgei] = NULL;
448		}
449	}
450
451	raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
452}
453
454static irqreturn_t hgei_interrupt(int irq, void *dev_id)
455{
456	int i;
457	unsigned long hgei_mask, flags;
458	struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei);
459
460	hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE);
461	csr_clear(CSR_HGEIE, hgei_mask);
462
463	raw_spin_lock_irqsave(&hgctrl->lock, flags);
464
465	for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) {
466		if (hgctrl->owners[i])
467			kvm_vcpu_kick(hgctrl->owners[i]);
468	}
469
470	raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
471
472	put_cpu_ptr(&aia_hgei);
473	return IRQ_HANDLED;
474}
475
476static int aia_hgei_init(void)
477{
478	int cpu, rc;
479	struct irq_domain *domain;
480	struct aia_hgei_control *hgctrl;
481
482	/* Initialize per-CPU guest external interrupt line management */
483	for_each_possible_cpu(cpu) {
484		hgctrl = per_cpu_ptr(&aia_hgei, cpu);
485		raw_spin_lock_init(&hgctrl->lock);
486		if (kvm_riscv_aia_nr_hgei) {
487			hgctrl->free_bitmap =
488				BIT(kvm_riscv_aia_nr_hgei + 1) - 1;
489			hgctrl->free_bitmap &= ~BIT(0);
490		} else
491			hgctrl->free_bitmap = 0;
492	}
493
494	/* Skip SGEI interrupt setup for zero guest external interrupts */
495	if (!kvm_riscv_aia_nr_hgei)
496		goto skip_sgei_interrupt;
497
498	/* Find INTC irq domain */
499	domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(),
500					  DOMAIN_BUS_ANY);
501	if (!domain) {
502		kvm_err("unable to find INTC domain\n");
503		return -ENOENT;
504	}
505
506	/* Map per-CPU SGEI interrupt from INTC domain */
507	hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT);
508	if (!hgei_parent_irq) {
509		kvm_err("unable to map SGEI IRQ\n");
510		return -ENOMEM;
511	}
512
513	/* Request per-CPU SGEI interrupt */
514	rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt,
515				"riscv-kvm", &aia_hgei);
516	if (rc) {
517		kvm_err("failed to request SGEI IRQ\n");
518		return rc;
519	}
520
521skip_sgei_interrupt:
522	return 0;
523}
524
525static void aia_hgei_exit(void)
526{
527	/* Do nothing for zero guest external interrupts */
528	if (!kvm_riscv_aia_nr_hgei)
529		return;
530
531	/* Free per-CPU SGEI interrupt */
532	free_percpu_irq(hgei_parent_irq, &aia_hgei);
533}
534
535void kvm_riscv_aia_enable(void)
536{
537	if (!kvm_riscv_aia_available())
538		return;
539
540	csr_write(CSR_HVICTL, aia_hvictl_value(false));
541	csr_write(CSR_HVIPRIO1, 0x0);
542	csr_write(CSR_HVIPRIO2, 0x0);
543#ifdef CONFIG_32BIT
544	csr_write(CSR_HVIPH, 0x0);
545	csr_write(CSR_HIDELEGH, 0x0);
546	csr_write(CSR_HVIPRIO1H, 0x0);
547	csr_write(CSR_HVIPRIO2H, 0x0);
548#endif
549
550	/* Enable per-CPU SGEI interrupt */
551	enable_percpu_irq(hgei_parent_irq,
552			  irq_get_trigger_type(hgei_parent_irq));
553	csr_set(CSR_HIE, BIT(IRQ_S_GEXT));
554	/* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */
555	if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF))
556		csr_set(CSR_HVIEN, BIT(IRQ_PMU_OVF));
557}
558
559void kvm_riscv_aia_disable(void)
560{
561	int i;
562	unsigned long flags;
563	struct kvm_vcpu *vcpu;
564	struct aia_hgei_control *hgctrl;
565
566	if (!kvm_riscv_aia_available())
567		return;
568	hgctrl = get_cpu_ptr(&aia_hgei);
569
570	if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF))
571		csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF));
572	/* Disable per-CPU SGEI interrupt */
573	csr_clear(CSR_HIE, BIT(IRQ_S_GEXT));
574	disable_percpu_irq(hgei_parent_irq);
575
576	csr_write(CSR_HVICTL, aia_hvictl_value(false));
577
578	raw_spin_lock_irqsave(&hgctrl->lock, flags);
579
580	for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) {
581		vcpu = hgctrl->owners[i];
582		if (!vcpu)
583			continue;
584
585		/*
586		 * We release hgctrl->lock before notifying IMSIC
587		 * so that we don't have lock ordering issues.
588		 */
589		raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
590
591		/* Notify IMSIC */
592		kvm_riscv_vcpu_aia_imsic_release(vcpu);
593
594		/*
595		 * Wakeup VCPU if it was blocked so that it can
596		 * run on other HARTs
597		 */
598		if (csr_read(CSR_HGEIE) & BIT(i)) {
599			csr_clear(CSR_HGEIE, BIT(i));
600			kvm_vcpu_kick(vcpu);
601		}
602
603		raw_spin_lock_irqsave(&hgctrl->lock, flags);
604	}
605
606	raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
607
608	put_cpu_ptr(&aia_hgei);
609}
610
611int kvm_riscv_aia_init(void)
612{
613	int rc;
614	const struct imsic_global_config *gc;
615
616	if (!riscv_isa_extension_available(NULL, SxAIA))
617		return -ENODEV;
618	gc = imsic_get_global_config();
619
620	/* Figure-out number of bits in HGEIE */
621	csr_write(CSR_HGEIE, -1UL);
622	kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE));
623	csr_write(CSR_HGEIE, 0);
624	if (kvm_riscv_aia_nr_hgei)
625		kvm_riscv_aia_nr_hgei--;
626
627	/*
628	 * Number of usable HGEI lines should be minimum of per-HART
629	 * IMSIC guest files and number of bits in HGEIE
630	 */
631	if (gc)
632		kvm_riscv_aia_nr_hgei = min((ulong)kvm_riscv_aia_nr_hgei,
633					    BIT(gc->guest_index_bits) - 1);
634	else
635		kvm_riscv_aia_nr_hgei = 0;
636
637	/* Find number of guest MSI IDs */
638	kvm_riscv_aia_max_ids = IMSIC_MAX_ID;
639	if (gc && kvm_riscv_aia_nr_hgei)
640		kvm_riscv_aia_max_ids = gc->nr_guest_ids + 1;
641
642	/* Initialize guest external interrupt line management */
643	rc = aia_hgei_init();
644	if (rc)
645		return rc;
646
647	/* Register device operations */
648	rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops,
649				     KVM_DEV_TYPE_RISCV_AIA);
650	if (rc) {
651		aia_hgei_exit();
652		return rc;
653	}
654
655	/* Enable KVM AIA support */
656	static_branch_enable(&kvm_riscv_aia_available);
657
658	return 0;
659}
660
661void kvm_riscv_aia_exit(void)
662{
663	if (!kvm_riscv_aia_available())
664		return;
665
666	/* Unregister device operations */
667	kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA);
668
669	/* Cleanup the HGEI state */
670	aia_hgei_exit();
671}