vc-shared.c

  1// SPDX-License-Identifier: GPL-2.0
  2
  3#ifndef __BOOT_COMPRESSED
  4#define has_cpuflag(f)                  cpu_feature_enabled(f)
  5#endif
  6
  7static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt,
  8					    unsigned long exit_code)
  9{
 10	unsigned int opcode = (unsigned int)ctxt->insn.opcode.value;
 11	u8 modrm = ctxt->insn.modrm.value;
 12
 13	switch (exit_code) {
 14
 15	case SVM_EXIT_IOIO:
 16	case SVM_EXIT_NPF:
 17		/* handled separately */
 18		return ES_OK;
 19
 20	case SVM_EXIT_CPUID:
 21		if (opcode == 0xa20f)
 22			return ES_OK;
 23		break;
 24
 25	case SVM_EXIT_INVD:
 26		if (opcode == 0x080f)
 27			return ES_OK;
 28		break;
 29
 30	case SVM_EXIT_MONITOR:
 31		/* MONITOR and MONITORX instructions generate the same error code */
 32		if (opcode == 0x010f && (modrm == 0xc8 || modrm == 0xfa))
 33			return ES_OK;
 34		break;
 35
 36	case SVM_EXIT_MWAIT:
 37		/* MWAIT and MWAITX instructions generate the same error code */
 38		if (opcode == 0x010f && (modrm == 0xc9 || modrm == 0xfb))
 39			return ES_OK;
 40		break;
 41
 42	case SVM_EXIT_MSR:
 43		/* RDMSR */
 44		if (opcode == 0x320f ||
 45		/* WRMSR */
 46		    opcode == 0x300f)
 47			return ES_OK;
 48		break;
 49
 50	case SVM_EXIT_RDPMC:
 51		if (opcode == 0x330f)
 52			return ES_OK;
 53		break;
 54
 55	case SVM_EXIT_RDTSC:
 56		if (opcode == 0x310f)
 57			return ES_OK;
 58		break;
 59
 60	case SVM_EXIT_RDTSCP:
 61		if (opcode == 0x010f && modrm == 0xf9)
 62			return ES_OK;
 63		break;
 64
 65	case SVM_EXIT_READ_DR7:
 66		if (opcode == 0x210f &&
 67		    X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
 68			return ES_OK;
 69		break;
 70
 71	case SVM_EXIT_VMMCALL:
 72		if (opcode == 0x010f && modrm == 0xd9)
 73			return ES_OK;
 74
 75		break;
 76
 77	case SVM_EXIT_WRITE_DR7:
 78		if (opcode == 0x230f &&
 79		    X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
 80			return ES_OK;
 81		break;
 82
 83	case SVM_EXIT_WBINVD:
 84		if (opcode == 0x90f)
 85			return ES_OK;
 86		break;
 87
 88	default:
 89		break;
 90	}
 91
 92	sev_printk(KERN_ERR "Wrong/unhandled opcode bytes: 0x%x, exit_code: 0x%lx, rIP: 0x%lx\n",
 93		   opcode, exit_code, ctxt->regs->ip);
 94
 95	return ES_UNSUPPORTED;
 96}
 97
 98static bool vc_decoding_needed(unsigned long exit_code)
 99{
100	/* Exceptions don't require to decode the instruction */
101	return !(exit_code >= SVM_EXIT_EXCP_BASE &&
102		 exit_code <= SVM_EXIT_LAST_EXCP);
103}
104
105static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
106				      struct pt_regs *regs,
107				      unsigned long exit_code)
108{
109	enum es_result ret = ES_OK;
110
111	memset(ctxt, 0, sizeof(*ctxt));
112	ctxt->regs = regs;
113
114	if (vc_decoding_needed(exit_code))
115		ret = vc_decode_insn(ctxt);
116
117	return ret;
118}
119
120static void vc_finish_insn(struct es_em_ctxt *ctxt)
121{
122	ctxt->regs->ip += ctxt->insn.length;
123}
124
125static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
126					   unsigned long address,
127					   bool write)
128{
129	if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
130		ctxt->fi.vector     = X86_TRAP_PF;
131		ctxt->fi.error_code = X86_PF_USER;
132		ctxt->fi.cr2        = address;
133		if (write)
134			ctxt->fi.error_code |= X86_PF_WRITE;
135
136		return ES_EXCEPTION;
137	}
138
139	return ES_OK;
140}
141
142static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
143					  void *src, char *buf,
144					  unsigned int data_size,
145					  unsigned int count,
146					  bool backwards)
147{
148	int i, b = backwards ? -1 : 1;
149	unsigned long address = (unsigned long)src;
150	enum es_result ret;
151
152	ret = vc_insn_string_check(ctxt, address, false);
153	if (ret != ES_OK)
154		return ret;
155
156	for (i = 0; i < count; i++) {
157		void *s = src + (i * data_size * b);
158		char *d = buf + (i * data_size);
159
160		ret = vc_read_mem(ctxt, s, d, data_size);
161		if (ret != ES_OK)
162			break;
163	}
164
165	return ret;
166}
167
168static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
169					   void *dst, char *buf,
170					   unsigned int data_size,
171					   unsigned int count,
172					   bool backwards)
173{
174	int i, s = backwards ? -1 : 1;
175	unsigned long address = (unsigned long)dst;
176	enum es_result ret;
177
178	ret = vc_insn_string_check(ctxt, address, true);
179	if (ret != ES_OK)
180		return ret;
181
182	for (i = 0; i < count; i++) {
183		void *d = dst + (i * data_size * s);
184		char *b = buf + (i * data_size);
185
186		ret = vc_write_mem(ctxt, d, b, data_size);
187		if (ret != ES_OK)
188			break;
189	}
190
191	return ret;
192}
193
194#define IOIO_TYPE_STR  BIT(2)
195#define IOIO_TYPE_IN   1
196#define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
197#define IOIO_TYPE_OUT  0
198#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
199
200#define IOIO_REP       BIT(3)
201
202#define IOIO_ADDR_64   BIT(9)
203#define IOIO_ADDR_32   BIT(8)
204#define IOIO_ADDR_16   BIT(7)
205
206#define IOIO_DATA_32   BIT(6)
207#define IOIO_DATA_16   BIT(5)
208#define IOIO_DATA_8    BIT(4)
209
210#define IOIO_SEG_ES    (0 << 10)
211#define IOIO_SEG_DS    (3 << 10)
212
213static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
214{
215	struct insn *insn = &ctxt->insn;
216	size_t size;
217	u64 port;
218
219	*exitinfo = 0;
220
221	switch (insn->opcode.bytes[0]) {
222	/* INS opcodes */
223	case 0x6c:
224	case 0x6d:
225		*exitinfo |= IOIO_TYPE_INS;
226		*exitinfo |= IOIO_SEG_ES;
227		port	   = ctxt->regs->dx & 0xffff;
228		break;
229
230	/* OUTS opcodes */
231	case 0x6e:
232	case 0x6f:
233		*exitinfo |= IOIO_TYPE_OUTS;
234		*exitinfo |= IOIO_SEG_DS;
235		port	   = ctxt->regs->dx & 0xffff;
236		break;
237
238	/* IN immediate opcodes */
239	case 0xe4:
240	case 0xe5:
241		*exitinfo |= IOIO_TYPE_IN;
242		port	   = (u8)insn->immediate.value & 0xffff;
243		break;
244
245	/* OUT immediate opcodes */
246	case 0xe6:
247	case 0xe7:
248		*exitinfo |= IOIO_TYPE_OUT;
249		port	   = (u8)insn->immediate.value & 0xffff;
250		break;
251
252	/* IN register opcodes */
253	case 0xec:
254	case 0xed:
255		*exitinfo |= IOIO_TYPE_IN;
256		port	   = ctxt->regs->dx & 0xffff;
257		break;
258
259	/* OUT register opcodes */
260	case 0xee:
261	case 0xef:
262		*exitinfo |= IOIO_TYPE_OUT;
263		port	   = ctxt->regs->dx & 0xffff;
264		break;
265
266	default:
267		return ES_DECODE_FAILED;
268	}
269
270	*exitinfo |= port << 16;
271
272	switch (insn->opcode.bytes[0]) {
273	case 0x6c:
274	case 0x6e:
275	case 0xe4:
276	case 0xe6:
277	case 0xec:
278	case 0xee:
279		/* Single byte opcodes */
280		*exitinfo |= IOIO_DATA_8;
281		size       = 1;
282		break;
283	default:
284		/* Length determined by instruction parsing */
285		*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
286						     : IOIO_DATA_32;
287		size       = (insn->opnd_bytes == 2) ? 2 : 4;
288	}
289
290	switch (insn->addr_bytes) {
291	case 2:
292		*exitinfo |= IOIO_ADDR_16;
293		break;
294	case 4:
295		*exitinfo |= IOIO_ADDR_32;
296		break;
297	case 8:
298		*exitinfo |= IOIO_ADDR_64;
299		break;
300	}
301
302	if (insn_has_rep_prefix(insn))
303		*exitinfo |= IOIO_REP;
304
305	return vc_ioio_check(ctxt, (u16)port, size);
306}
307
308static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
309{
310	struct pt_regs *regs = ctxt->regs;
311	u64 exit_info_1, exit_info_2;
312	enum es_result ret;
313
314	ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
315	if (ret != ES_OK)
316		return ret;
317
318	if (exit_info_1 & IOIO_TYPE_STR) {
319
320		/* (REP) INS/OUTS */
321
322		bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
323		unsigned int io_bytes, exit_bytes;
324		unsigned int ghcb_count, op_count;
325		unsigned long es_base;
326		u64 sw_scratch;
327
328		/*
329		 * For the string variants with rep prefix the amount of in/out
330		 * operations per #VC exception is limited so that the kernel
331		 * has a chance to take interrupts and re-schedule while the
332		 * instruction is emulated.
333		 */
334		io_bytes   = (exit_info_1 >> 4) & 0x7;
335		ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
336
337		op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
338		exit_info_2 = min(op_count, ghcb_count);
339		exit_bytes  = exit_info_2 * io_bytes;
340
341		es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
342
343		/* Read bytes of OUTS into the shared buffer */
344		if (!(exit_info_1 & IOIO_TYPE_IN)) {
345			ret = vc_insn_string_read(ctxt,
346					       (void *)(es_base + regs->si),
347					       ghcb->shared_buffer, io_bytes,
348					       exit_info_2, df);
349			if (ret)
350				return ret;
351		}
352
353		/*
354		 * Issue an VMGEXIT to the HV to consume the bytes from the
355		 * shared buffer or to have it write them into the shared buffer
356		 * depending on the instruction: OUTS or INS.
357		 */
358		sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
359		ghcb_set_sw_scratch(ghcb, sw_scratch);
360		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
361					  exit_info_1, exit_info_2);
362		if (ret != ES_OK)
363			return ret;
364
365		/* Read bytes from shared buffer into the guest's destination. */
366		if (exit_info_1 & IOIO_TYPE_IN) {
367			ret = vc_insn_string_write(ctxt,
368						   (void *)(es_base + regs->di),
369						   ghcb->shared_buffer, io_bytes,
370						   exit_info_2, df);
371			if (ret)
372				return ret;
373
374			if (df)
375				regs->di -= exit_bytes;
376			else
377				regs->di += exit_bytes;
378		} else {
379			if (df)
380				regs->si -= exit_bytes;
381			else
382				regs->si += exit_bytes;
383		}
384
385		if (exit_info_1 & IOIO_REP)
386			regs->cx -= exit_info_2;
387
388		ret = regs->cx ? ES_RETRY : ES_OK;
389
390	} else {
391
392		/* IN/OUT into/from rAX */
393
394		int bits = (exit_info_1 & 0x70) >> 1;
395		u64 rax = 0;
396
397		if (!(exit_info_1 & IOIO_TYPE_IN))
398			rax = lower_bits(regs->ax, bits);
399
400		ghcb_set_rax(ghcb, rax);
401
402		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
403		if (ret != ES_OK)
404			return ret;
405
406		if (exit_info_1 & IOIO_TYPE_IN) {
407			if (!ghcb_rax_is_valid(ghcb))
408				return ES_VMM_ERROR;
409			regs->ax = lower_bits(ghcb->save.rax, bits);
410		}
411	}
412
413	return ret;
414}
415
416enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
417{
418	u32 ret;
419
420	ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);
421	if (!ret)
422		return ES_OK;
423
424	if (ret == 1) {
425		u64 info = ghcb->save.sw_exit_info_2;
426		unsigned long v = info & SVM_EVTINJ_VEC_MASK;
427
428		/* Check if exception information from hypervisor is sane. */
429		if ((info & SVM_EVTINJ_VALID) &&
430		    ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
431		    ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
432			ctxt->fi.vector = v;
433
434			if (info & SVM_EVTINJ_VALID_ERR)
435				ctxt->fi.error_code = info >> 32;
436
437			return ES_EXCEPTION;
438		}
439	}
440
441	return ES_VMM_ERROR;
442}
443
444enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
445				   struct es_em_ctxt *ctxt,
446				   u64 exit_code, u64 exit_info_1,
447				   u64 exit_info_2)
448{
449	/* Fill in protocol and format specifiers */
450	ghcb->protocol_version = ghcb_version;
451	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
452
453	ghcb_set_sw_exit_code(ghcb, exit_code);
454	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
455	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
456
457	sev_es_wr_ghcb_msr(__pa(ghcb));
458	VMGEXIT();
459
460	return verify_exception_info(ghcb, ctxt);
461}
462
463static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
464{
465	u32 cr4 = native_read_cr4();
466	int ret;
467
468	ghcb_set_rax(ghcb, leaf->fn);
469	ghcb_set_rcx(ghcb, leaf->subfn);
470
471	if (cr4 & X86_CR4_OSXSAVE)
472		/* Safe to read xcr0 */
473		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
474	else
475		/* xgetbv will cause #UD - use reset value for xcr0 */
476		ghcb_set_xcr0(ghcb, 1);
477
478	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
479	if (ret != ES_OK)
480		return ret;
481
482	if (!(ghcb_rax_is_valid(ghcb) &&
483	      ghcb_rbx_is_valid(ghcb) &&
484	      ghcb_rcx_is_valid(ghcb) &&
485	      ghcb_rdx_is_valid(ghcb)))
486		return ES_VMM_ERROR;
487
488	leaf->eax = ghcb->save.rax;
489	leaf->ebx = ghcb->save.rbx;
490	leaf->ecx = ghcb->save.rcx;
491	leaf->edx = ghcb->save.rdx;
492
493	return ES_OK;
494}
495
496struct cpuid_ctx {
497	struct ghcb *ghcb;
498	struct es_em_ctxt *ctxt;
499};
500
501static void snp_cpuid_hv_ghcb(void *p, struct cpuid_leaf *leaf)
502{
503	struct cpuid_ctx *ctx = p;
504
505	if (__sev_cpuid_hv_ghcb(ctx->ghcb, ctx->ctxt, leaf))
506		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
507}
508
509static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
510{
511	struct cpuid_ctx ctx = { ghcb, ctxt };
512	struct pt_regs *regs = ctxt->regs;
513	struct cpuid_leaf leaf;
514	int ret;
515
516	leaf.fn = regs->ax;
517	leaf.subfn = regs->cx;
518	ret = snp_cpuid(snp_cpuid_hv_ghcb, &ctx, &leaf);
519	if (!ret) {
520		regs->ax = leaf.eax;
521		regs->bx = leaf.ebx;
522		regs->cx = leaf.ecx;
523		regs->dx = leaf.edx;
524	}
525
526	return ret;
527}
528
529static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
530				      struct es_em_ctxt *ctxt)
531{
532	struct pt_regs *regs = ctxt->regs;
533	u32 cr4 = native_read_cr4();
534	enum es_result ret;
535	int snp_cpuid_ret;
536
537	snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt);
538	if (!snp_cpuid_ret)
539		return ES_OK;
540	if (snp_cpuid_ret != -EOPNOTSUPP)
541		return ES_VMM_ERROR;
542
543	ghcb_set_rax(ghcb, regs->ax);
544	ghcb_set_rcx(ghcb, regs->cx);
545
546	if (cr4 & X86_CR4_OSXSAVE)
547		/* Safe to read xcr0 */
548		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
549	else
550		/* xgetbv will cause #GP - use reset value for xcr0 */
551		ghcb_set_xcr0(ghcb, 1);
552
553	if (has_cpuflag(X86_FEATURE_SHSTK) && regs->ax == 0xd && regs->cx == 1) {
554		struct msr m;
555
556		raw_rdmsr(MSR_IA32_XSS, &m);
557		ghcb_set_xss(ghcb, m.q);
558	}
559
560	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
561	if (ret != ES_OK)
562		return ret;
563
564	if (!(ghcb_rax_is_valid(ghcb) &&
565	      ghcb_rbx_is_valid(ghcb) &&
566	      ghcb_rcx_is_valid(ghcb) &&
567	      ghcb_rdx_is_valid(ghcb)))
568		return ES_VMM_ERROR;
569
570	regs->ax = ghcb->save.rax;
571	regs->bx = ghcb->save.rbx;
572	regs->cx = ghcb->save.rcx;
573	regs->dx = ghcb->save.rdx;
574
575	return ES_OK;
576}
577
578static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
579				      struct es_em_ctxt *ctxt,
580				      unsigned long exit_code)
581{
582	bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
583	enum es_result ret;
584
585	/*
586	 * The hypervisor should not be intercepting RDTSC/RDTSCP when Secure
587	 * TSC is enabled. A #VC exception will be generated if the RDTSC/RDTSCP
588	 * instructions are being intercepted. If this should occur and Secure
589	 * TSC is enabled, guest execution should be terminated as the guest
590	 * cannot rely on the TSC value provided by the hypervisor.
591	 */
592	if (sev_status & MSR_AMD64_SNP_SECURE_TSC)
593		return ES_VMM_ERROR;
594
595	ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
596	if (ret != ES_OK)
597		return ret;
598
599	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
600	     (!rdtscp || ghcb_rcx_is_valid(ghcb))))
601		return ES_VMM_ERROR;
602
603	ctxt->regs->ax = ghcb->save.rax;
604	ctxt->regs->dx = ghcb->save.rdx;
605	if (rdtscp)
606		ctxt->regs->cx = ghcb->save.rcx;
607
608	return ES_OK;
609}
610
611void snp_register_ghcb_early(unsigned long paddr)
612{
613	unsigned long pfn = paddr >> PAGE_SHIFT;
614	u64 val;
615
616	sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
617	VMGEXIT();
618
619	val = sev_es_rd_ghcb_msr();
620
621	/* If the response GPA is not ours then abort the guest */
622	if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||
623	    (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
624		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
625}
626
627bool __init sev_es_check_cpu_features(void)
628{
629	if (!has_cpuflag(X86_FEATURE_RDRAND)) {
630		error("RDRAND instruction not supported - no trusted source of randomness available\n");
631		return false;
632	}
633
634	return true;
635}
636
637bool sev_es_negotiate_protocol(void)
638{
639	u64 val;
640
641	/* Do the GHCB protocol version negotiation */
642	sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
643	VMGEXIT();
644	val = sev_es_rd_ghcb_msr();
645
646	if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
647		return false;
648
649	if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
650	    GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
651		return false;
652
653	ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
654
655	return true;
656}