exit.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
  4 */
  5
  6#include <linux/err.h>
  7#include <linux/errno.h>
  8#include <linux/kvm_host.h>
  9#include <linux/module.h>
 10#include <linux/preempt.h>
 11#include <linux/vmalloc.h>
 12#include <trace/events/kvm.h>
 13#include <asm/fpu.h>
 14#include <asm/inst.h>
 15#include <asm/loongarch.h>
 16#include <asm/mmzone.h>
 17#include <asm/numa.h>
 18#include <asm/time.h>
 19#include <asm/tlb.h>
 20#include <asm/kvm_csr.h>
 21#include <asm/kvm_vcpu.h>
 22#include "trace.h"
 23
 24static int kvm_emu_cpucfg(struct kvm_vcpu *vcpu, larch_inst inst)
 25{
 26	int rd, rj;
 27	unsigned int index, ret;
 28
 29	if (inst.reg2_format.opcode != cpucfg_op)
 30		return EMULATE_FAIL;
 31
 32	rd = inst.reg2_format.rd;
 33	rj = inst.reg2_format.rj;
 34	++vcpu->stat.cpucfg_exits;
 35	index = vcpu->arch.gprs[rj];
 36
 37	/*
 38	 * By LoongArch Reference Manual 2.2.10.5
 39	 * Return value is 0 for undefined CPUCFG index
 40	 *
 41	 * Disable preemption since hw gcsr is accessed
 42	 */
 43	preempt_disable();
 44	switch (index) {
 45	case 0 ... (KVM_MAX_CPUCFG_REGS - 1):
 46		vcpu->arch.gprs[rd] = vcpu->arch.cpucfg[index];
 47		break;
 48	case CPUCFG_KVM_SIG:
 49		/* CPUCFG emulation between 0x40000000 -- 0x400000ff */
 50		vcpu->arch.gprs[rd] = *(unsigned int *)KVM_SIGNATURE;
 51		break;
 52	case CPUCFG_KVM_FEATURE:
 53		ret = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK;
 54		vcpu->arch.gprs[rd] = ret;
 55		break;
 56	default:
 57		vcpu->arch.gprs[rd] = 0;
 58		break;
 59	}
 60	preempt_enable();
 61
 62	return EMULATE_DONE;
 63}
 64
 65static unsigned long kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid)
 66{
 67	unsigned long val = 0;
 68	struct loongarch_csrs *csr = vcpu->arch.csr;
 69
 70	/*
 71	 * From LoongArch Reference Manual Volume 1 Chapter 4.2.1
 72	 * For undefined CSR id, return value is 0
 73	 */
 74	if (get_gcsr_flag(csrid) & SW_GCSR)
 75		val = kvm_read_sw_gcsr(csr, csrid);
 76	else
 77		pr_warn_once("Unsupported csrrd 0x%x with pc %lx\n", csrid, vcpu->arch.pc);
 78
 79	return val;
 80}
 81
 82static unsigned long kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, unsigned long val)
 83{
 84	unsigned long old = 0;
 85	struct loongarch_csrs *csr = vcpu->arch.csr;
 86
 87	if (get_gcsr_flag(csrid) & SW_GCSR) {
 88		old = kvm_read_sw_gcsr(csr, csrid);
 89		kvm_write_sw_gcsr(csr, csrid, val);
 90	} else
 91		pr_warn_once("Unsupported csrwr 0x%x with pc %lx\n", csrid, vcpu->arch.pc);
 92
 93	return old;
 94}
 95
 96static unsigned long kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid,
 97				unsigned long csr_mask, unsigned long val)
 98{
 99	unsigned long old = 0;
100	struct loongarch_csrs *csr = vcpu->arch.csr;
101
102	if (get_gcsr_flag(csrid) & SW_GCSR) {
103		old = kvm_read_sw_gcsr(csr, csrid);
104		val = (old & ~csr_mask) | (val & csr_mask);
105		kvm_write_sw_gcsr(csr, csrid, val);
106		old = old & csr_mask;
107	} else
108		pr_warn_once("Unsupported csrxchg 0x%x with pc %lx\n", csrid, vcpu->arch.pc);
109
110	return old;
111}
112
113static int kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst)
114{
115	unsigned int rd, rj, csrid;
116	unsigned long csr_mask, val = 0;
117
118	/*
119	 * CSR value mask imm
120	 * rj = 0 means csrrd
121	 * rj = 1 means csrwr
122	 * rj != 0,1 means csrxchg
123	 */
124	rd = inst.reg2csr_format.rd;
125	rj = inst.reg2csr_format.rj;
126	csrid = inst.reg2csr_format.csr;
127
128	if (csrid >= LOONGARCH_CSR_PERFCTRL0 && csrid <= vcpu->arch.max_pmu_csrid) {
129		if (kvm_guest_has_pmu(&vcpu->arch)) {
130			vcpu->arch.pc -= 4;
131			kvm_make_request(KVM_REQ_PMU, vcpu);
132			return EMULATE_DONE;
133		}
134	}
135
136	/* Process CSR ops */
137	switch (rj) {
138	case 0: /* process csrrd */
139		val = kvm_emu_read_csr(vcpu, csrid);
140		vcpu->arch.gprs[rd] = val;
141		break;
142	case 1: /* process csrwr */
143		val = vcpu->arch.gprs[rd];
144		val = kvm_emu_write_csr(vcpu, csrid, val);
145		vcpu->arch.gprs[rd] = val;
146		break;
147	default: /* process csrxchg */
148		val = vcpu->arch.gprs[rd];
149		csr_mask = vcpu->arch.gprs[rj];
150		val = kvm_emu_xchg_csr(vcpu, csrid, csr_mask, val);
151		vcpu->arch.gprs[rd] = val;
152	}
153
154	return EMULATE_DONE;
155}
156
157int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu)
158{
159	int idx, ret;
160	unsigned long *val;
161	u32 addr, rd, rj, opcode;
162
163	/*
164	 * Each IOCSR with different opcode
165	 */
166	rd = inst.reg2_format.rd;
167	rj = inst.reg2_format.rj;
168	opcode = inst.reg2_format.opcode;
169	addr = vcpu->arch.gprs[rj];
170	run->iocsr_io.phys_addr = addr;
171	run->iocsr_io.is_write = 0;
172	val = &vcpu->arch.gprs[rd];
173
174	/* LoongArch is Little endian */
175	switch (opcode) {
176	case iocsrrdb_op:
177		run->iocsr_io.len = 1;
178		break;
179	case iocsrrdh_op:
180		run->iocsr_io.len = 2;
181		break;
182	case iocsrrdw_op:
183		run->iocsr_io.len = 4;
184		break;
185	case iocsrrdd_op:
186		run->iocsr_io.len = 8;
187		break;
188	case iocsrwrb_op:
189		run->iocsr_io.len = 1;
190		run->iocsr_io.is_write = 1;
191		break;
192	case iocsrwrh_op:
193		run->iocsr_io.len = 2;
194		run->iocsr_io.is_write = 1;
195		break;
196	case iocsrwrw_op:
197		run->iocsr_io.len = 4;
198		run->iocsr_io.is_write = 1;
199		break;
200	case iocsrwrd_op:
201		run->iocsr_io.len = 8;
202		run->iocsr_io.is_write = 1;
203		break;
204	default:
205		return EMULATE_FAIL;
206	}
207
208	if (run->iocsr_io.is_write) {
209		idx = srcu_read_lock(&vcpu->kvm->srcu);
210		ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, run->iocsr_io.len, val);
211		srcu_read_unlock(&vcpu->kvm->srcu, idx);
212		if (ret == 0)
213			ret = EMULATE_DONE;
214		else {
215			ret = EMULATE_DO_IOCSR;
216			/* Save data and let user space to write it */
217			memcpy(run->iocsr_io.data, val, run->iocsr_io.len);
218		}
219		trace_kvm_iocsr(KVM_TRACE_IOCSR_WRITE, run->iocsr_io.len, addr, val);
220	} else {
221		vcpu->arch.io_gpr = rd; /* Set register id for iocsr read completion */
222		idx = srcu_read_lock(&vcpu->kvm->srcu);
223		ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr,
224				      run->iocsr_io.len, run->iocsr_io.data);
225		srcu_read_unlock(&vcpu->kvm->srcu, idx);
226		if (ret == 0) {
227			kvm_complete_iocsr_read(vcpu, run);
228			ret = EMULATE_DONE;
229		} else
230			ret = EMULATE_DO_IOCSR;
231		trace_kvm_iocsr(KVM_TRACE_IOCSR_READ, run->iocsr_io.len, addr, NULL);
232	}
233
234	return ret;
235}
236
237int kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run)
238{
239	enum emulation_result er = EMULATE_DONE;
240	unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
241
242	switch (run->iocsr_io.len) {
243	case 1:
244		*gpr = *(s8 *)run->iocsr_io.data;
245		break;
246	case 2:
247		*gpr = *(s16 *)run->iocsr_io.data;
248		break;
249	case 4:
250		*gpr = *(s32 *)run->iocsr_io.data;
251		break;
252	case 8:
253		*gpr = *(s64 *)run->iocsr_io.data;
254		break;
255	default:
256		kvm_err("Bad IOCSR length: %d, addr is 0x%lx\n",
257				run->iocsr_io.len, vcpu->arch.badv);
258		er = EMULATE_FAIL;
259		break;
260	}
261
262	return er;
263}
264
265int kvm_emu_idle(struct kvm_vcpu *vcpu)
266{
267	++vcpu->stat.idle_exits;
268	trace_kvm_exit_idle(vcpu, KVM_TRACE_EXIT_IDLE);
269
270	if (!kvm_arch_vcpu_runnable(vcpu))
271		kvm_vcpu_halt(vcpu);
272
273	return EMULATE_DONE;
274}
275
276static int kvm_trap_handle_gspr(struct kvm_vcpu *vcpu)
277{
278	unsigned long curr_pc;
279	larch_inst inst;
280	enum emulation_result er = EMULATE_DONE;
281	struct kvm_run *run = vcpu->run;
282
283	/* Fetch the instruction */
284	inst.word = vcpu->arch.badi;
285	curr_pc = vcpu->arch.pc;
286	update_pc(&vcpu->arch);
287
288	trace_kvm_exit_gspr(vcpu, inst.word);
289	er = EMULATE_FAIL;
290	switch (((inst.word >> 24) & 0xff)) {
291	case 0x0: /* CPUCFG GSPR */
292		trace_kvm_exit_cpucfg(vcpu, KVM_TRACE_EXIT_CPUCFG);
293		er = kvm_emu_cpucfg(vcpu, inst);
294		break;
295	case 0x4: /* CSR{RD,WR,XCHG} GSPR */
296		trace_kvm_exit_csr(vcpu, KVM_TRACE_EXIT_CSR);
297		er = kvm_handle_csr(vcpu, inst);
298		break;
299	case 0x6: /* Cache, Idle and IOCSR GSPR */
300		switch (((inst.word >> 22) & 0x3ff)) {
301		case 0x18: /* Cache GSPR */
302			er = EMULATE_DONE;
303			trace_kvm_exit_cache(vcpu, KVM_TRACE_EXIT_CACHE);
304			break;
305		case 0x19: /* Idle/IOCSR GSPR */
306			switch (((inst.word >> 15) & 0x1ffff)) {
307			case 0xc90: /* IOCSR GSPR */
308				er = kvm_emu_iocsr(inst, run, vcpu);
309				break;
310			case 0xc91: /* Idle GSPR */
311				er = kvm_emu_idle(vcpu);
312				break;
313			default:
314				er = EMULATE_FAIL;
315				break;
316			}
317			break;
318		default:
319			er = EMULATE_FAIL;
320			break;
321		}
322		break;
323	default:
324		er = EMULATE_FAIL;
325		break;
326	}
327
328	/* Rollback PC only if emulation was unsuccessful */
329	if (er == EMULATE_FAIL) {
330		kvm_err("[%#lx]%s: unsupported gspr instruction 0x%08x\n",
331			curr_pc, __func__, inst.word);
332
333		kvm_arch_vcpu_dump_regs(vcpu);
334		vcpu->arch.pc = curr_pc;
335	}
336
337	return er;
338}
339
340/*
341 * Trigger GSPR:
342 * 1) Execute CPUCFG instruction;
343 * 2) Execute CACOP/IDLE instructions;
344 * 3) Access to unimplemented CSRs/IOCSRs.
345 */
346static int kvm_handle_gspr(struct kvm_vcpu *vcpu, int ecode)
347{
348	int ret = RESUME_GUEST;
349	enum emulation_result er = EMULATE_DONE;
350
351	er = kvm_trap_handle_gspr(vcpu);
352
353	if (er == EMULATE_DONE) {
354		ret = RESUME_GUEST;
355	} else if (er == EMULATE_DO_MMIO) {
356		vcpu->run->exit_reason = KVM_EXIT_MMIO;
357		ret = RESUME_HOST;
358	} else if (er == EMULATE_DO_IOCSR) {
359		vcpu->run->exit_reason = KVM_EXIT_LOONGARCH_IOCSR;
360		ret = RESUME_HOST;
361	} else {
362		kvm_queue_exception(vcpu, EXCCODE_INE, 0);
363		ret = RESUME_GUEST;
364	}
365
366	return ret;
367}
368
369int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst)
370{
371	int idx, ret;
372	unsigned int op8, opcode, rd;
373	struct kvm_run *run = vcpu->run;
374
375	run->mmio.phys_addr = vcpu->arch.badv;
376	vcpu->mmio_needed = 2;	/* signed */
377	op8 = (inst.word >> 24) & 0xff;
378	ret = EMULATE_DO_MMIO;
379
380	switch (op8) {
381	case 0x24 ... 0x27:	/* ldptr.w/d process */
382		rd = inst.reg2i14_format.rd;
383		opcode = inst.reg2i14_format.opcode;
384
385		switch (opcode) {
386		case ldptrw_op:
387			run->mmio.len = 4;
388			break;
389		case ldptrd_op:
390			run->mmio.len = 8;
391			break;
392		default:
393			break;
394		}
395		break;
396	case 0x28 ... 0x2e:	/* ld.b/h/w/d, ld.bu/hu/wu process */
397		rd = inst.reg2i12_format.rd;
398		opcode = inst.reg2i12_format.opcode;
399
400		switch (opcode) {
401		case ldb_op:
402			run->mmio.len = 1;
403			break;
404		case ldbu_op:
405			vcpu->mmio_needed = 1;	/* unsigned */
406			run->mmio.len = 1;
407			break;
408		case ldh_op:
409			run->mmio.len = 2;
410			break;
411		case ldhu_op:
412			vcpu->mmio_needed = 1;	/* unsigned */
413			run->mmio.len = 2;
414			break;
415		case ldw_op:
416			run->mmio.len = 4;
417			break;
418		case ldwu_op:
419			vcpu->mmio_needed = 1;	/* unsigned */
420			run->mmio.len = 4;
421			break;
422		case ldd_op:
423			run->mmio.len = 8;
424			break;
425		default:
426			ret = EMULATE_FAIL;
427			break;
428		}
429		break;
430	case 0x38:	/* ldx.b/h/w/d, ldx.bu/hu/wu process */
431		rd = inst.reg3_format.rd;
432		opcode = inst.reg3_format.opcode;
433
434		switch (opcode) {
435		case ldxb_op:
436			run->mmio.len = 1;
437			break;
438		case ldxbu_op:
439			run->mmio.len = 1;
440			vcpu->mmio_needed = 1;	/* unsigned */
441			break;
442		case ldxh_op:
443			run->mmio.len = 2;
444			break;
445		case ldxhu_op:
446			run->mmio.len = 2;
447			vcpu->mmio_needed = 1;	/* unsigned */
448			break;
449		case ldxw_op:
450			run->mmio.len = 4;
451			break;
452		case ldxwu_op:
453			run->mmio.len = 4;
454			vcpu->mmio_needed = 1;	/* unsigned */
455			break;
456		case ldxd_op:
457			run->mmio.len = 8;
458			break;
459		default:
460			ret = EMULATE_FAIL;
461			break;
462		}
463		break;
464	default:
465		ret = EMULATE_FAIL;
466	}
467
468	if (ret == EMULATE_DO_MMIO) {
469		trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len, run->mmio.phys_addr, NULL);
470
471		vcpu->arch.io_gpr = rd; /* Set for kvm_complete_mmio_read() use */
472
473		/*
474		 * If mmio device such as PCH-PIC is emulated in KVM,
475		 * it need not return to user space to handle the mmio
476		 * exception.
477		 */
478		idx = srcu_read_lock(&vcpu->kvm->srcu);
479		ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, vcpu->arch.badv,
480				      run->mmio.len, run->mmio.data);
481		srcu_read_unlock(&vcpu->kvm->srcu, idx);
482		if (!ret) {
483			kvm_complete_mmio_read(vcpu, run);
484			update_pc(&vcpu->arch);
485			vcpu->mmio_needed = 0;
486			return EMULATE_DONE;
487		}
488
489		run->mmio.is_write = 0;
490		vcpu->mmio_is_write = 0;
491		return EMULATE_DO_MMIO;
492	}
493
494	kvm_err("Read not supported Inst=0x%08x @%lx BadVaddr:%#lx\n",
495			inst.word, vcpu->arch.pc, vcpu->arch.badv);
496	kvm_arch_vcpu_dump_regs(vcpu);
497	vcpu->mmio_needed = 0;
498
499	return ret;
500}
501
502int kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run)
503{
504	enum emulation_result er = EMULATE_DONE;
505	unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
506
507	/* Update with new PC */
508	update_pc(&vcpu->arch);
509	switch (run->mmio.len) {
510	case 1:
511		if (vcpu->mmio_needed == 2)
512			*gpr = *(s8 *)run->mmio.data;
513		else
514			*gpr = *(u8 *)run->mmio.data;
515		break;
516	case 2:
517		if (vcpu->mmio_needed == 2)
518			*gpr = *(s16 *)run->mmio.data;
519		else
520			*gpr = *(u16 *)run->mmio.data;
521		break;
522	case 4:
523		if (vcpu->mmio_needed == 2)
524			*gpr = *(s32 *)run->mmio.data;
525		else
526			*gpr = *(u32 *)run->mmio.data;
527		break;
528	case 8:
529		*gpr = *(s64 *)run->mmio.data;
530		break;
531	default:
532		kvm_err("Bad MMIO length: %d, addr is 0x%lx\n",
533				run->mmio.len, vcpu->arch.badv);
534		er = EMULATE_FAIL;
535		break;
536	}
537
538	trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len,
539			run->mmio.phys_addr, run->mmio.data);
540
541	return er;
542}
543
544int kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst)
545{
546	int idx, ret;
547	unsigned int rd, op8, opcode;
548	unsigned long curr_pc, rd_val = 0;
549	struct kvm_run *run = vcpu->run;
550	void *data = run->mmio.data;
551
552	/*
553	 * Update PC and hold onto current PC in case there is
554	 * an error and we want to rollback the PC
555	 */
556	curr_pc = vcpu->arch.pc;
557	update_pc(&vcpu->arch);
558
559	op8 = (inst.word >> 24) & 0xff;
560	run->mmio.phys_addr = vcpu->arch.badv;
561	ret = EMULATE_DO_MMIO;
562	switch (op8) {
563	case 0x24 ... 0x27:	/* stptr.w/d process */
564		rd = inst.reg2i14_format.rd;
565		opcode = inst.reg2i14_format.opcode;
566
567		switch (opcode) {
568		case stptrw_op:
569			run->mmio.len = 4;
570			*(unsigned int *)data = vcpu->arch.gprs[rd];
571			break;
572		case stptrd_op:
573			run->mmio.len = 8;
574			*(unsigned long *)data = vcpu->arch.gprs[rd];
575			break;
576		default:
577			ret = EMULATE_FAIL;
578			break;
579		}
580		break;
581	case 0x28 ... 0x2e:	/* st.b/h/w/d  process */
582		rd = inst.reg2i12_format.rd;
583		opcode = inst.reg2i12_format.opcode;
584		rd_val = vcpu->arch.gprs[rd];
585
586		switch (opcode) {
587		case stb_op:
588			run->mmio.len = 1;
589			*(unsigned char *)data = rd_val;
590			break;
591		case sth_op:
592			run->mmio.len = 2;
593			*(unsigned short *)data = rd_val;
594			break;
595		case stw_op:
596			run->mmio.len = 4;
597			*(unsigned int *)data = rd_val;
598			break;
599		case std_op:
600			run->mmio.len = 8;
601			*(unsigned long *)data = rd_val;
602			break;
603		default:
604			ret = EMULATE_FAIL;
605			break;
606		}
607		break;
608	case 0x38:	/* stx.b/h/w/d process */
609		rd = inst.reg3_format.rd;
610		opcode = inst.reg3_format.opcode;
611
612		switch (opcode) {
613		case stxb_op:
614			run->mmio.len = 1;
615			*(unsigned char *)data = vcpu->arch.gprs[rd];
616			break;
617		case stxh_op:
618			run->mmio.len = 2;
619			*(unsigned short *)data = vcpu->arch.gprs[rd];
620			break;
621		case stxw_op:
622			run->mmio.len = 4;
623			*(unsigned int *)data = vcpu->arch.gprs[rd];
624			break;
625		case stxd_op:
626			run->mmio.len = 8;
627			*(unsigned long *)data = vcpu->arch.gprs[rd];
628			break;
629		default:
630			ret = EMULATE_FAIL;
631			break;
632		}
633		break;
634	default:
635		ret = EMULATE_FAIL;
636	}
637
638	if (ret == EMULATE_DO_MMIO) {
639		trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, run->mmio.len, run->mmio.phys_addr, data);
640
641		/*
642		 * If mmio device such as PCH-PIC is emulated in KVM,
643		 * it need not return to user space to handle the mmio
644		 * exception.
645		 */
646		idx = srcu_read_lock(&vcpu->kvm->srcu);
647		ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, run->mmio.len, data);
648		srcu_read_unlock(&vcpu->kvm->srcu, idx);
649		if (!ret)
650			return EMULATE_DONE;
651
652		run->mmio.is_write = 1;
653		vcpu->mmio_needed = 1;
654		vcpu->mmio_is_write = 1;
655		return EMULATE_DO_MMIO;
656	}
657
658	vcpu->arch.pc = curr_pc;
659	kvm_err("Write not supported Inst=0x%08x @%lx BadVaddr:%#lx\n",
660			inst.word, vcpu->arch.pc, vcpu->arch.badv);
661	kvm_arch_vcpu_dump_regs(vcpu);
662	/* Rollback PC if emulation was unsuccessful */
663
664	return ret;
665}
666
667static int kvm_handle_rdwr_fault(struct kvm_vcpu *vcpu, bool write, int ecode)
668{
669	int ret;
670	larch_inst inst;
671	enum emulation_result er = EMULATE_DONE;
672	struct kvm_run *run = vcpu->run;
673	unsigned long badv = vcpu->arch.badv;
674
675	/* Inject ADE exception if exceed max GPA size */
676	if (unlikely(badv >= vcpu->kvm->arch.gpa_size)) {
677		kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM);
678		return RESUME_GUEST;
679	}
680
681	ret = kvm_handle_mm_fault(vcpu, badv, write, ecode);
682	if (ret) {
683		/* Treat as MMIO */
684		inst.word = vcpu->arch.badi;
685		if (write) {
686			er = kvm_emu_mmio_write(vcpu, inst);
687		} else {
688			/* A code fetch fault doesn't count as an MMIO */
689			if (kvm_is_ifetch_fault(&vcpu->arch)) {
690				kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEF);
691				return RESUME_GUEST;
692			}
693
694			er = kvm_emu_mmio_read(vcpu, inst);
695		}
696	}
697
698	if (er == EMULATE_DONE) {
699		ret = RESUME_GUEST;
700	} else if (er == EMULATE_DO_MMIO) {
701		run->exit_reason = KVM_EXIT_MMIO;
702		ret = RESUME_HOST;
703	} else {
704		kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM);
705		ret = RESUME_GUEST;
706	}
707
708	return ret;
709}
710
711static int kvm_handle_read_fault(struct kvm_vcpu *vcpu, int ecode)
712{
713	return kvm_handle_rdwr_fault(vcpu, false, ecode);
714}
715
716static int kvm_handle_write_fault(struct kvm_vcpu *vcpu, int ecode)
717{
718	return kvm_handle_rdwr_fault(vcpu, true, ecode);
719}
720
721int kvm_complete_user_service(struct kvm_vcpu *vcpu, struct kvm_run *run)
722{
723	update_pc(&vcpu->arch);
724	kvm_write_reg(vcpu, LOONGARCH_GPR_A0, run->hypercall.ret);
725
726	return 0;
727}
728
729/**
730 * kvm_handle_fpu_disabled() - Guest used fpu however it is disabled at host
731 * @vcpu:	Virtual CPU context.
732 * @ecode:	Exception code.
733 *
734 * Handle when the guest attempts to use fpu which hasn't been allowed
735 * by the root context.
736 */
737static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu, int ecode)
738{
739	struct kvm_run *run = vcpu->run;
740
741	if (!kvm_guest_has_fpu(&vcpu->arch)) {
742		kvm_queue_exception(vcpu, EXCCODE_INE, 0);
743		return RESUME_GUEST;
744	}
745
746	/*
747	 * If guest FPU not present, the FPU operation should have been
748	 * treated as a reserved instruction!
749	 * If FPU already in use, we shouldn't get this at all.
750	 */
751	if (WARN_ON(vcpu->arch.aux_inuse & KVM_LARCH_FPU)) {
752		kvm_err("%s internal error\n", __func__);
753		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
754		return RESUME_HOST;
755	}
756
757	kvm_own_fpu(vcpu);
758
759	return RESUME_GUEST;
760}
761
762static long kvm_save_notify(struct kvm_vcpu *vcpu)
763{
764	unsigned long id, data;
765
766	id   = kvm_read_reg(vcpu, LOONGARCH_GPR_A1);
767	data = kvm_read_reg(vcpu, LOONGARCH_GPR_A2);
768	switch (id) {
769	case BIT(KVM_FEATURE_STEAL_TIME):
770		if (data & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID))
771			return KVM_HCALL_INVALID_PARAMETER;
772
773		vcpu->arch.st.guest_addr = data;
774		if (!(data & KVM_STEAL_PHYS_VALID))
775			return 0;
776
777		vcpu->arch.st.last_steal = current->sched_info.run_delay;
778		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
779		return 0;
780	default:
781		return KVM_HCALL_INVALID_CODE;
782	}
783}
784
785/*
786 * kvm_handle_lsx_disabled() - Guest used LSX while disabled in root.
787 * @vcpu:      Virtual CPU context.
788 * @ecode:	Exception code.
789 *
790 * Handle when the guest attempts to use LSX when it is disabled in the root
791 * context.
792 */
793static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu, int ecode)
794{
795	if (kvm_own_lsx(vcpu))
796		kvm_queue_exception(vcpu, EXCCODE_INE, 0);
797
798	return RESUME_GUEST;
799}
800
801/*
802 * kvm_handle_lasx_disabled() - Guest used LASX while disabled in root.
803 * @vcpu:	Virtual CPU context.
804 * @ecode:	Exception code.
805 *
806 * Handle when the guest attempts to use LASX when it is disabled in the root
807 * context.
808 */
809static int kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu, int ecode)
810{
811	if (kvm_own_lasx(vcpu))
812		kvm_queue_exception(vcpu, EXCCODE_INE, 0);
813
814	return RESUME_GUEST;
815}
816
817static int kvm_handle_lbt_disabled(struct kvm_vcpu *vcpu, int ecode)
818{
819	if (kvm_own_lbt(vcpu))
820		kvm_queue_exception(vcpu, EXCCODE_INE, 0);
821
822	return RESUME_GUEST;
823}
824
825static void kvm_send_pv_ipi(struct kvm_vcpu *vcpu)
826{
827	unsigned int min, cpu;
828	struct kvm_vcpu *dest;
829	DECLARE_BITMAP(ipi_bitmap, BITS_PER_LONG * 2) = {
830		kvm_read_reg(vcpu, LOONGARCH_GPR_A1),
831		kvm_read_reg(vcpu, LOONGARCH_GPR_A2)
832	};
833
834	min = kvm_read_reg(vcpu, LOONGARCH_GPR_A3);
835	for_each_set_bit(cpu, ipi_bitmap, BITS_PER_LONG * 2) {
836		dest = kvm_get_vcpu_by_cpuid(vcpu->kvm, cpu + min);
837		if (!dest)
838			continue;
839
840		/* Send SWI0 to dest vcpu to emulate IPI interrupt */
841		kvm_queue_irq(dest, INT_SWI0);
842		kvm_vcpu_kick(dest);
843	}
844}
845
846/*
847 * Hypercall emulation always return to guest, Caller should check retval.
848 */
849static void kvm_handle_service(struct kvm_vcpu *vcpu)
850{
851	long ret = KVM_HCALL_INVALID_CODE;
852	unsigned long func = kvm_read_reg(vcpu, LOONGARCH_GPR_A0);
853
854	switch (func) {
855	case KVM_HCALL_FUNC_IPI:
856		if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_IPI)) {
857			kvm_send_pv_ipi(vcpu);
858			ret = KVM_HCALL_SUCCESS;
859		}
860		break;
861	case KVM_HCALL_FUNC_NOTIFY:
862		if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME))
863			ret = kvm_save_notify(vcpu);
864		break;
865	default:
866		break;
867	}
868
869	kvm_write_reg(vcpu, LOONGARCH_GPR_A0, ret);
870}
871
872static int kvm_handle_hypercall(struct kvm_vcpu *vcpu, int ecode)
873{
874	int ret;
875	larch_inst inst;
876	unsigned int code;
877
878	inst.word = vcpu->arch.badi;
879	code = inst.reg0i15_format.immediate;
880	ret = RESUME_GUEST;
881
882	switch (code) {
883	case KVM_HCALL_SERVICE:
884		vcpu->stat.hypercall_exits++;
885		kvm_handle_service(vcpu);
886		break;
887	case KVM_HCALL_USER_SERVICE:
888		if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_USER_HCALL)) {
889			kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE);
890			break;
891		}
892
893		vcpu->stat.hypercall_exits++;
894		vcpu->run->exit_reason = KVM_EXIT_HYPERCALL;
895		vcpu->run->hypercall.nr = KVM_HCALL_USER_SERVICE;
896		vcpu->run->hypercall.args[0] = kvm_read_reg(vcpu, LOONGARCH_GPR_A0);
897		vcpu->run->hypercall.args[1] = kvm_read_reg(vcpu, LOONGARCH_GPR_A1);
898		vcpu->run->hypercall.args[2] = kvm_read_reg(vcpu, LOONGARCH_GPR_A2);
899		vcpu->run->hypercall.args[3] = kvm_read_reg(vcpu, LOONGARCH_GPR_A3);
900		vcpu->run->hypercall.args[4] = kvm_read_reg(vcpu, LOONGARCH_GPR_A4);
901		vcpu->run->hypercall.args[5] = kvm_read_reg(vcpu, LOONGARCH_GPR_A5);
902		vcpu->run->hypercall.flags = 0;
903		/*
904		 * Set invalid return value by default, let user-mode VMM modify it.
905		 */
906		vcpu->run->hypercall.ret = KVM_HCALL_INVALID_CODE;
907		ret = RESUME_HOST;
908		break;
909	case KVM_HCALL_SWDBG:
910		/* KVM_HCALL_SWDBG only in effective when SW_BP is enabled */
911		if (vcpu->guest_debug & KVM_GUESTDBG_SW_BP_MASK) {
912			vcpu->run->exit_reason = KVM_EXIT_DEBUG;
913			ret = RESUME_HOST;
914			break;
915		}
916		fallthrough;
917	default:
918		/* Treat it as noop intruction, only set return value */
919		kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE);
920		break;
921	}
922
923	if (ret == RESUME_GUEST)
924		update_pc(&vcpu->arch);
925
926	return ret;
927}
928
929/*
930 * LoongArch KVM callback handling for unimplemented guest exiting
931 */
932static int kvm_fault_ni(struct kvm_vcpu *vcpu, int ecode)
933{
934	unsigned int inst;
935	unsigned long badv;
936
937	/* Fetch the instruction */
938	inst = vcpu->arch.badi;
939	badv = vcpu->arch.badv;
940	kvm_err("ECode: %d PC=%#lx Inst=0x%08x BadVaddr=%#lx ESTAT=%#lx\n",
941			ecode, vcpu->arch.pc, inst, badv, read_gcsr_estat());
942	kvm_arch_vcpu_dump_regs(vcpu);
943	kvm_queue_exception(vcpu, EXCCODE_INE, 0);
944
945	return RESUME_GUEST;
946}
947
948static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = {
949	[0 ... EXCCODE_INT_START - 1]	= kvm_fault_ni,
950	[EXCCODE_TLBI]			= kvm_handle_read_fault,
951	[EXCCODE_TLBL]			= kvm_handle_read_fault,
952	[EXCCODE_TLBS]			= kvm_handle_write_fault,
953	[EXCCODE_TLBM]			= kvm_handle_write_fault,
954	[EXCCODE_FPDIS]			= kvm_handle_fpu_disabled,
955	[EXCCODE_LSXDIS]		= kvm_handle_lsx_disabled,
956	[EXCCODE_LASXDIS]		= kvm_handle_lasx_disabled,
957	[EXCCODE_BTDIS]			= kvm_handle_lbt_disabled,
958	[EXCCODE_GSPR]			= kvm_handle_gspr,
959	[EXCCODE_HVC]			= kvm_handle_hypercall,
960};
961
962int kvm_handle_fault(struct kvm_vcpu *vcpu, int fault)
963{
964	return kvm_fault_tables[fault](vcpu, fault);
965}