types.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * FPU data structures:
  4 */
  5#ifndef _ASM_X86_FPU_TYPES_H
  6#define _ASM_X86_FPU_TYPES_H
  7
  8#include <asm/page_types.h>
  9
 10/*
 11 * The legacy x87 FPU state format, as saved by FSAVE and
 12 * restored by the FRSTOR instructions:
 13 */
 14struct fregs_state {
 15	u32			cwd;	/* FPU Control Word		*/
 16	u32			swd;	/* FPU Status Word		*/
 17	u32			twd;	/* FPU Tag Word			*/
 18	u32			fip;	/* FPU IP Offset		*/
 19	u32			fcs;	/* FPU IP Selector		*/
 20	u32			foo;	/* FPU Operand Pointer Offset	*/
 21	u32			fos;	/* FPU Operand Pointer Selector	*/
 22
 23	/* 8*10 bytes for each FP-reg = 80 bytes:			*/
 24	u32			st_space[20];
 25
 26	/* Software status information [not touched by FSAVE]:		*/
 27	u32			status;
 28};
 29
 30/*
 31 * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and
 32 * restored by the FXRSTOR instructions. It's similar to the FSAVE
 33 * format, but differs in some areas, plus has extensions at
 34 * the end for the XMM registers.
 35 */
 36struct fxregs_state {
 37	u16			cwd; /* Control Word			*/
 38	u16			swd; /* Status Word			*/
 39	u16			twd; /* Tag Word			*/
 40	u16			fop; /* Last Instruction Opcode		*/
 41	union {
 42		struct {
 43			u64	rip; /* Instruction Pointer		*/
 44			u64	rdp; /* Data Pointer			*/
 45		};
 46		struct {
 47			u32	fip; /* FPU IP Offset			*/
 48			u32	fcs; /* FPU IP Selector			*/
 49			u32	foo; /* FPU Operand Offset		*/
 50			u32	fos; /* FPU Operand Selector		*/
 51		};
 52	};
 53	u32			mxcsr;		/* MXCSR Register State */
 54	u32			mxcsr_mask;	/* MXCSR Mask		*/
 55
 56	/* 8*16 bytes for each FP-reg = 128 bytes:			*/
 57	u32			st_space[32];
 58
 59	/* 16*16 bytes for each XMM-reg = 256 bytes:			*/
 60	u32			xmm_space[64];
 61
 62	u32			padding[12];
 63
 64	union {
 65		u32		padding1[12];
 66		u32		sw_reserved[12];
 67	};
 68
 69} __attribute__((aligned(16)));
 70
 71/* Default value for fxregs_state.mxcsr: */
 72#define MXCSR_DEFAULT		0x1f80
 73
 74/* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */
 75#define MXCSR_AND_FLAGS_SIZE sizeof(u64)
 76
 77/*
 78 * Software based FPU emulation state. This is arbitrary really,
 79 * it matches the x87 format to make it easier to understand:
 80 */
 81struct swregs_state {
 82	u32			cwd;
 83	u32			swd;
 84	u32			twd;
 85	u32			fip;
 86	u32			fcs;
 87	u32			foo;
 88	u32			fos;
 89	/* 8*10 bytes for each FP-reg = 80 bytes: */
 90	u32			st_space[20];
 91	u8			ftop;
 92	u8			changed;
 93	u8			lookahead;
 94	u8			no_update;
 95	u8			rm;
 96	u8			alimit;
 97	struct math_emu_info	*info;
 98	u32			entry_eip;
 99};
100
101/*
102 * List of XSAVE features Linux knows about:
103 */
104enum xfeature {
105	XFEATURE_FP,
106	XFEATURE_SSE,
107	/*
108	 * Values above here are "legacy states".
109	 * Those below are "extended states".
110	 */
111	XFEATURE_YMM,
112	XFEATURE_BNDREGS,
113	XFEATURE_BNDCSR,
114	XFEATURE_OPMASK,
115	XFEATURE_ZMM_Hi256,
116	XFEATURE_Hi16_ZMM,
117	XFEATURE_PT_UNIMPLEMENTED_SO_FAR,
118	XFEATURE_PKRU,
119	XFEATURE_PASID,
120	XFEATURE_CET_USER,
121	XFEATURE_CET_KERNEL,
122	XFEATURE_RSRVD_COMP_13,
123	XFEATURE_RSRVD_COMP_14,
124	XFEATURE_LBR,
125	XFEATURE_RSRVD_COMP_16,
126	XFEATURE_XTILE_CFG,
127	XFEATURE_XTILE_DATA,
128	XFEATURE_APX,
129
130	XFEATURE_MAX,
131};
132
133#define XFEATURE_MASK_FP		(1 << XFEATURE_FP)
134#define XFEATURE_MASK_SSE		(1 << XFEATURE_SSE)
135#define XFEATURE_MASK_YMM		(1 << XFEATURE_YMM)
136#define XFEATURE_MASK_BNDREGS		(1 << XFEATURE_BNDREGS)
137#define XFEATURE_MASK_BNDCSR		(1 << XFEATURE_BNDCSR)
138#define XFEATURE_MASK_OPMASK		(1 << XFEATURE_OPMASK)
139#define XFEATURE_MASK_ZMM_Hi256		(1 << XFEATURE_ZMM_Hi256)
140#define XFEATURE_MASK_Hi16_ZMM		(1 << XFEATURE_Hi16_ZMM)
141#define XFEATURE_MASK_PT		(1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
142#define XFEATURE_MASK_PKRU		(1 << XFEATURE_PKRU)
143#define XFEATURE_MASK_PASID		(1 << XFEATURE_PASID)
144#define XFEATURE_MASK_CET_USER		(1 << XFEATURE_CET_USER)
145#define XFEATURE_MASK_CET_KERNEL	(1 << XFEATURE_CET_KERNEL)
146#define XFEATURE_MASK_LBR		(1 << XFEATURE_LBR)
147#define XFEATURE_MASK_XTILE_CFG		(1 << XFEATURE_XTILE_CFG)
148#define XFEATURE_MASK_XTILE_DATA	(1 << XFEATURE_XTILE_DATA)
149#define XFEATURE_MASK_APX		(1 << XFEATURE_APX)
150
151#define XFEATURE_MASK_FPSSE		(XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
152#define XFEATURE_MASK_AVX512		(XFEATURE_MASK_OPMASK \
153					 | XFEATURE_MASK_ZMM_Hi256 \
154					 | XFEATURE_MASK_Hi16_ZMM)
155
156#ifdef CONFIG_X86_64
157# define XFEATURE_MASK_XTILE		(XFEATURE_MASK_XTILE_DATA \
158					 | XFEATURE_MASK_XTILE_CFG)
159#else
160# define XFEATURE_MASK_XTILE		(0)
161#endif
162
163#define FIRST_EXTENDED_XFEATURE	XFEATURE_YMM
164
165struct reg_128_bit {
166	u8      regbytes[128/8];
167};
168struct reg_256_bit {
169	u8	regbytes[256/8];
170};
171struct reg_512_bit {
172	u8	regbytes[512/8];
173};
174struct reg_1024_byte {
175	u8	regbytes[1024];
176};
177
178/*
179 * State component 2:
180 *
181 * There are 16x 256-bit AVX registers named YMM0-YMM15.
182 * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15)
183 * and are stored in 'struct fxregs_state::xmm_space[]' in the
184 * "legacy" area.
185 *
186 * The high 128 bits are stored here.
187 */
188struct ymmh_struct {
189	struct reg_128_bit              hi_ymm[16];
190} __packed;
191
192/* Intel MPX support: */
193
194struct mpx_bndreg {
195	u64				lower_bound;
196	u64				upper_bound;
197} __packed;
198/*
199 * State component 3 is used for the 4 128-bit bounds registers
200 */
201struct mpx_bndreg_state {
202	struct mpx_bndreg		bndreg[4];
203} __packed;
204
205/*
206 * State component 4 is used for the 64-bit user-mode MPX
207 * configuration register BNDCFGU and the 64-bit MPX status
208 * register BNDSTATUS.  We call the pair "BNDCSR".
209 */
210struct mpx_bndcsr {
211	u64				bndcfgu;
212	u64				bndstatus;
213} __packed;
214
215/*
216 * The BNDCSR state is padded out to be 64-bytes in size.
217 */
218struct mpx_bndcsr_state {
219	union {
220		struct mpx_bndcsr		bndcsr;
221		u8				pad_to_64_bytes[64];
222	};
223} __packed;
224
225/* AVX-512 Components: */
226
227/*
228 * State component 5 is used for the 8 64-bit opmask registers
229 * k0-k7 (opmask state).
230 */
231struct avx_512_opmask_state {
232	u64				opmask_reg[8];
233} __packed;
234
235/*
236 * State component 6 is used for the upper 256 bits of the
237 * registers ZMM0-ZMM15. These 16 256-bit values are denoted
238 * ZMM0_H-ZMM15_H (ZMM_Hi256 state).
239 */
240struct avx_512_zmm_uppers_state {
241	struct reg_256_bit		zmm_upper[16];
242} __packed;
243
244/*
245 * State component 7 is used for the 16 512-bit registers
246 * ZMM16-ZMM31 (Hi16_ZMM state).
247 */
248struct avx_512_hi16_state {
249	struct reg_512_bit		hi16_zmm[16];
250} __packed;
251
252/*
253 * State component 9: 32-bit PKRU register.  The state is
254 * 8 bytes long but only 4 bytes is used currently.
255 */
256struct pkru_state {
257	u32				pkru;
258	u32				pad;
259} __packed;
260
261/*
262 * State component 11 is Control-flow Enforcement user states
263 */
264struct cet_user_state {
265	/* user control-flow settings */
266	u64 user_cet;
267	/* user shadow stack pointer */
268	u64 user_ssp;
269};
270
271/*
272 * State component 12 is Control-flow Enforcement supervisor states.
273 * This state includes SSP pointers for privilege levels 0 through 2.
274 */
275struct cet_supervisor_state {
276	u64 pl0_ssp;
277	u64 pl1_ssp;
278	u64 pl2_ssp;
279} __packed;
280
281/*
282 * State component 15: Architectural LBR configuration state.
283 * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
284 */
285
286struct lbr_entry {
287	u64 from;
288	u64 to;
289	u64 info;
290};
291
292struct arch_lbr_state {
293	u64 lbr_ctl;
294	u64 lbr_depth;
295	u64 ler_from;
296	u64 ler_to;
297	u64 ler_info;
298	struct lbr_entry		entries[];
299};
300
301/*
302 * State component 17: 64-byte tile configuration register.
303 */
304struct xtile_cfg {
305	u64				tcfg[8];
306} __packed;
307
308/*
309 * State component 18: 1KB tile data register.
310 * Each register represents 16 64-byte rows of the matrix
311 * data. But the number of registers depends on the actual
312 * implementation.
313 */
314struct xtile_data {
315	struct reg_1024_byte		tmm;
316} __packed;
317
318/*
319 * State component 19: 8B extended general purpose register.
320 */
321struct apx_state {
322	u64				egpr[16];
323} __packed;
324
325/*
326 * State component 10 is supervisor state used for context-switching the
327 * PASID state.
328 */
329struct ia32_pasid_state {
330	u64 pasid;
331} __packed;
332
333struct xstate_header {
334	u64				xfeatures;
335	u64				xcomp_bv;
336	u64				reserved[6];
337} __attribute__((packed));
338
339/*
340 * xstate_header.xcomp_bv[63] indicates that the extended_state_area
341 * is in compacted format.
342 */
343#define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63)
344
345/*
346 * This is our most modern FPU state format, as saved by the XSAVE
347 * and restored by the XRSTOR instructions.
348 *
349 * It consists of a legacy fxregs portion, an xstate header and
350 * subsequent areas as defined by the xstate header.  Not all CPUs
351 * support all the extensions, so the size of the extended area
352 * can vary quite a bit between CPUs.
353 */
354struct xregs_state {
355	struct fxregs_state		i387;
356	struct xstate_header		header;
357	u8				extended_state_area[];
358} __attribute__ ((packed, aligned (64)));
359
360/*
361 * This is a union of all the possible FPU state formats
362 * put together, so that we can pick the right one runtime.
363 *
364 * The size of the structure is determined by the largest
365 * member - which is the xsave area.  The padding is there
366 * to ensure that statically-allocated task_structs (just
367 * the init_task today) have enough space.
368 */
369union fpregs_state {
370	struct fregs_state		fsave;
371	struct fxregs_state		fxsave;
372	struct swregs_state		soft;
373	struct xregs_state		xsave;
374	u8 __padding[PAGE_SIZE];
375};
376
377struct fpstate {
378	/* @kernel_size: The size of the kernel register image */
379	unsigned int		size;
380
381	/* @user_size: The size in non-compacted UABI format */
382	unsigned int		user_size;
383
384	/* @xfeatures:		xfeatures for which the storage is sized */
385	u64			xfeatures;
386
387	/* @user_xfeatures:	xfeatures valid in UABI buffers */
388	u64			user_xfeatures;
389
390	/* @xfd:		xfeatures disabled to trap userspace use. */
391	u64			xfd;
392
393	/* @is_valloc:		Indicator for dynamically allocated state */
394	unsigned int		is_valloc	: 1;
395
396	/* @is_guest:		Indicator for guest state (KVM) */
397	unsigned int		is_guest	: 1;
398
399	/*
400	 * @is_confidential:	Indicator for KVM confidential mode.
401	 *			The FPU registers are restored by the
402	 *			vmentry firmware from encrypted guest
403	 *			memory. On vmexit the FPU registers are
404	 *			saved by firmware to encrypted guest memory
405	 *			and the registers are scrubbed before
406	 *			returning to the host. So there is no
407	 *			content which is worth saving and restoring.
408	 *			The fpstate has to be there so that
409	 *			preemption and softirq FPU usage works
410	 *			without special casing.
411	 */
412	unsigned int		is_confidential	: 1;
413
414	/* @in_use:		State is in use */
415	unsigned int		in_use		: 1;
416
417	/* @regs: The register state union for all supported formats */
418	union fpregs_state	regs;
419
420	/* @regs is dynamically sized! Don't add anything after @regs! */
421} __aligned(64);
422
423#define FPU_GUEST_PERM_LOCKED		BIT_ULL(63)
424
425struct fpu_state_perm {
426	/*
427	 * @__state_perm:
428	 *
429	 * This bitmap indicates the permission for state components
430	 * available to a thread group, including both user and supervisor
431	 * components and software-defined bits like FPU_GUEST_PERM_LOCKED.
432	 * The permission prctl() sets the enabled state bits in
433	 * thread_group_leader()->thread.fpu.
434	 *
435	 * All run time operations use the per thread information in the
436	 * currently active fpu.fpstate which contains the xfeature masks
437	 * and sizes for kernel and user space.
438	 *
439	 * This master permission field is only to be used when
440	 * task.fpu.fpstate based checks fail to validate whether the task
441	 * is allowed to expand its xfeatures set which requires to
442	 * allocate a larger sized fpstate buffer.
443	 *
444	 * Do not access this field directly.  Use the provided helper
445	 * function. Unlocked access is possible for quick checks.
446	 */
447	u64				__state_perm;
448
449	/*
450	 * @__state_size:
451	 *
452	 * The size required for @__state_perm. Only valid to access
453	 * with sighand locked.
454	 */
455	unsigned int			__state_size;
456
457	/*
458	 * @__user_state_size:
459	 *
460	 * The size required for @__state_perm user part. Only valid to
461	 * access with sighand locked.
462	 */
463	unsigned int			__user_state_size;
464};
465
466/*
467 * Highest level per task FPU state data structure that
468 * contains the FPU register state plus various FPU
469 * state fields:
470 */
471struct fpu {
472	/*
473	 * @last_cpu:
474	 *
475	 * Records the last CPU on which this context was loaded into
476	 * FPU registers. (In the lazy-restore case we might be
477	 * able to reuse FPU registers across multiple context switches
478	 * this way, if no intermediate task used the FPU.)
479	 *
480	 * A value of -1 is used to indicate that the FPU state in context
481	 * memory is newer than the FPU state in registers, and that the
482	 * FPU state should be reloaded next time the task is run.
483	 */
484	unsigned int			last_cpu;
485
486	/*
487	 * @avx512_timestamp:
488	 *
489	 * Records the timestamp of AVX512 use during last context switch.
490	 */
491	unsigned long			avx512_timestamp;
492
493	/*
494	 * @fpstate:
495	 *
496	 * Pointer to the active struct fpstate. Initialized to
497	 * point at @__fpstate below.
498	 */
499	struct fpstate			*fpstate;
500
501	/*
502	 * @__task_fpstate:
503	 *
504	 * Pointer to an inactive struct fpstate. Initialized to NULL. Is
505	 * used only for KVM support to swap out the regular task fpstate.
506	 */
507	struct fpstate			*__task_fpstate;
508
509	/*
510	 * @perm:
511	 *
512	 * Permission related information
513	 */
514	struct fpu_state_perm		perm;
515
516	/*
517	 * @guest_perm:
518	 *
519	 * Permission related information for guest pseudo FPUs
520	 */
521	struct fpu_state_perm		guest_perm;
522
523	/*
524	 * @__fpstate:
525	 *
526	 * Initial in-memory storage for FPU registers which are saved in
527	 * context switch and when the kernel uses the FPU. The registers
528	 * are restored from this storage on return to user space if they
529	 * are not longer containing the tasks FPU register state.
530	 */
531	struct fpstate			__fpstate;
532	/*
533	 * WARNING: '__fpstate' is dynamically-sized.  Do not put
534	 * anything after it here.
535	 */
536};
537
538/*
539 * Guest pseudo FPU container
540 */
541struct fpu_guest {
542	/*
543	 * @xfeatures:			xfeature bitmap of features which are
544	 *				currently enabled for the guest vCPU.
545	 */
546	u64				xfeatures;
547
548	/*
549	 * @xfd_err:			Save the guest value.
550	 */
551	u64				xfd_err;
552
553	/*
554	 * @uabi_size:			Size required for save/restore
555	 */
556	unsigned int			uabi_size;
557
558	/*
559	 * @fpstate:			Pointer to the allocated guest fpstate
560	 */
561	struct fpstate			*fpstate;
562};
563
564/*
565 * FPU state configuration data for fpu_guest.
566 * Initialized at boot time. Read only after init.
567 */
568struct vcpu_fpu_config {
569	/*
570	 * @size:
571	 *
572	 * The default size of the register state buffer in guest FPUs.
573	 * Includes all supported features except independent managed
574	 * features and features which have to be requested by user space
575	 * before usage.
576	 */
577	unsigned int size;
578
579	/*
580	 * @features:
581	 *
582	 * The default supported features bitmap in guest FPUs. Does not
583	 * include independent managed features and features which have to
584	 * be requested by user space before usage.
585	 */
586	u64 features;
587};
588
589/*
590 * FPU state configuration data. Initialized at boot time. Read only after init.
591 */
592struct fpu_state_config {
593	/*
594	 * @max_size:
595	 *
596	 * The maximum size of the register state buffer. Includes all
597	 * supported features except independent managed features.
598	 */
599	unsigned int		max_size;
600
601	/*
602	 * @default_size:
603	 *
604	 * The default size of the register state buffer. Includes all
605	 * supported features except independent managed features,
606	 * guest-only features and features which have to be requested by
607	 * user space before usage.
608	 */
609	unsigned int		default_size;
610
611	/*
612	 * @max_features:
613	 *
614	 * The maximum supported features bitmap. Does not include
615	 * independent managed features.
616	 */
617	u64 max_features;
618
619	/*
620	 * @default_features:
621	 *
622	 * The default supported features bitmap. Does not include
623	 * independent managed features, guest-only features and features
624	 * which have to be requested by user space before usage.
625	 */
626	u64 default_features;
627	/*
628	 * @legacy_features:
629	 *
630	 * Features which can be reported back to user space
631	 * even without XSAVE support, i.e. legacy features FP + SSE
632	 */
633	u64 legacy_features;
634	/*
635	 * @independent_features:
636	 *
637	 * Features that are supported by XSAVES, but not managed as part of
638	 * the FPU core, such as LBR
639	 */
640	u64 independent_features;
641};
642
643/* FPU state configuration information */
644extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg;
645extern struct vcpu_fpu_config guest_default_cfg;
646
647#endif /* _ASM_X86_FPU_TYPES_H */