xstate.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef __X86_KERNEL_FPU_XSTATE_H
  3#define __X86_KERNEL_FPU_XSTATE_H
  4
  5#include <asm/cpufeature.h>
  6#include <asm/fpu/xstate.h>
  7#include <asm/fpu/xcr.h>
  8#include <asm/msr.h>
  9
 10#ifdef CONFIG_X86_64
 11DECLARE_PER_CPU(u64, xfd_state);
 12#endif
 13
 14static inline void xstate_init_xcomp_bv(struct xregs_state *xsave, u64 mask)
 15{
 16	/*
 17	 * XRSTORS requires these bits set in xcomp_bv, or it will
 18	 * trigger #GP:
 19	 */
 20	if (cpu_feature_enabled(X86_FEATURE_XCOMPACTED))
 21		xsave->header.xcomp_bv = mask | XCOMP_BV_COMPACTED_FORMAT;
 22}
 23
 24static inline u64 xstate_get_group_perm(bool guest)
 25{
 26	struct fpu *fpu = x86_task_fpu(current->group_leader);
 27	struct fpu_state_perm *perm;
 28
 29	/* Pairs with WRITE_ONCE() in xstate_request_perm() */
 30	perm = guest ? &fpu->guest_perm : &fpu->perm;
 31	return READ_ONCE(perm->__state_perm);
 32}
 33
 34static inline u64 xstate_get_host_group_perm(void)
 35{
 36	return xstate_get_group_perm(false);
 37}
 38
 39enum xstate_copy_mode {
 40	XSTATE_COPY_FP,
 41	XSTATE_COPY_FX,
 42	XSTATE_COPY_XSAVE,
 43};
 44
 45struct membuf;
 46extern void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate,
 47				      u64 xfeatures, u32 pkru_val,
 48				      enum xstate_copy_mode copy_mode);
 49extern void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk,
 50				    enum xstate_copy_mode mode);
 51extern int copy_uabi_from_kernel_to_xstate(struct fpstate *fpstate, const void *kbuf, u32 *pkru);
 52extern int copy_sigframe_from_user_to_xstate(struct task_struct *tsk, const void __user *ubuf);
 53
 54
 55extern void fpu__init_cpu_xstate(void);
 56extern void fpu__init_system_xstate(unsigned int legacy_size);
 57
 58extern void __user *get_xsave_addr_user(struct xregs_state __user *xsave, int xfeature_nr);
 59
 60static inline u64 xfeatures_mask_supervisor(void)
 61{
 62	return fpu_kernel_cfg.max_features & XFEATURE_MASK_SUPERVISOR_SUPPORTED;
 63}
 64
 65static inline u64 xfeatures_mask_independent(void)
 66{
 67	if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR))
 68		return fpu_kernel_cfg.independent_features & ~XFEATURE_MASK_LBR;
 69
 70	return fpu_kernel_cfg.independent_features;
 71}
 72
 73static inline int set_xfeature_in_sigframe(struct xregs_state __user *xbuf, u64 mask)
 74{
 75	u64 xfeatures;
 76	int err;
 77
 78	/* Read the xfeatures value already saved in the user buffer */
 79	err  = __get_user(xfeatures, &xbuf->header.xfeatures);
 80	xfeatures |= mask;
 81	err |= __put_user(xfeatures, &xbuf->header.xfeatures);
 82
 83	return err;
 84}
 85
 86/*
 87 * Update the value of PKRU register that was already pushed onto the signal frame.
 88 */
 89static inline int update_pkru_in_sigframe(struct xregs_state __user *buf, u32 pkru)
 90{
 91	int err;
 92
 93	if (unlikely(!cpu_feature_enabled(X86_FEATURE_OSPKE)))
 94		return 0;
 95
 96	/* Mark PKRU as in-use so that it is restored correctly. */
 97	err = set_xfeature_in_sigframe(buf, XFEATURE_MASK_PKRU);
 98	if (err)
 99		return err;
100
101	/* Update PKRU value in the userspace xsave buffer. */
102	return __put_user(pkru, (unsigned int __user *)get_xsave_addr_user(buf, XFEATURE_PKRU));
103}
104
105/* XSAVE/XRSTOR wrapper functions */
106
107#ifdef CONFIG_X86_64
108#define REX_SUFFIX	"64"
109#else
110#define REX_SUFFIX
111#endif
112
113#define XSAVE		"xsave" REX_SUFFIX " %[xa]"
114#define XSAVEOPT	"xsaveopt" REX_SUFFIX " %[xa]"
115#define XSAVEC		"xsavec" REX_SUFFIX " %[xa]"
116#define XSAVES		"xsaves" REX_SUFFIX " %[xa]"
117#define XRSTOR		"xrstor" REX_SUFFIX " %[xa]"
118#define XRSTORS		"xrstors" REX_SUFFIX " %[xa]"
119
120/*
121 * After this @err contains 0 on success or the trap number when the
122 * operation raises an exception.
123 *
124 * The [xa] input parameter below represents the struct xregs_state pointer
125 * and the asm symbolic name for the argument used in the XSAVE/XRSTOR insns
126 * above.
127 */
128#define XSTATE_OP(op, st, lmask, hmask, err)				\
129	asm volatile("1:" op "\n\t"					\
130		     "xor %[err], %[err]\n"				\
131		     "2:\n"						\
132		     _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_FAULT_MCE_SAFE)	\
133		     : [err] "=a" (err)					\
134		     : [xa] "m" (*(st)), "a" (lmask), "d" (hmask)	\
135		     : "memory")
136
137/*
138 * If XSAVES is enabled, it replaces XSAVEC because it supports supervisor
139 * states in addition to XSAVEC.
140 *
141 * Otherwise if XSAVEC is enabled, it replaces XSAVEOPT because it supports
142 * compacted storage format in addition to XSAVEOPT.
143 *
144 * Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT
145 * supports modified optimization which is not supported by XSAVE.
146 *
147 * Use XSAVE as a fallback.
148 */
149#define XSTATE_XSAVE(st, lmask, hmask, err)				\
150	asm volatile("1: " ALTERNATIVE_3(XSAVE,				\
151				   XSAVEOPT, X86_FEATURE_XSAVEOPT,	\
152				   XSAVEC,   X86_FEATURE_XSAVEC,	\
153				   XSAVES,   X86_FEATURE_XSAVES)	\
154		     "\n\t"						\
155		     "xor %[err], %[err]\n"				\
156		     "3:\n"						\
157		     _ASM_EXTABLE_TYPE_REG(1b, 3b, EX_TYPE_EFAULT_REG, %[err]) \
158		     : [err] "=r" (err)					\
159		     : [xa] "m" (*(st)), "a" (lmask), "d" (hmask)	\
160		     : "memory")
161
162/*
163 * Use XRSTORS to restore context if it is enabled. XRSTORS supports compact
164 * XSAVE area format.
165 */
166#define XSTATE_XRESTORE(st, lmask, hmask)				\
167	asm volatile("1: " ALTERNATIVE(XRSTOR,				\
168				 XRSTORS, X86_FEATURE_XSAVES)		\
169		     "\n"						\
170		     "3:\n"						\
171		     _ASM_EXTABLE_TYPE(1b, 3b, EX_TYPE_FPU_RESTORE)	\
172		     :							\
173		     : [xa] "m" (*(st)), "a" (lmask), "d" (hmask)	\
174		     : "memory")
175
176#if defined(CONFIG_X86_64) && defined(CONFIG_X86_DEBUG_FPU)
177extern void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor);
178#else
179static inline void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor) { }
180#endif
181
182#ifdef CONFIG_X86_64
183static inline void xfd_set_state(u64 xfd)
184{
185	wrmsrq(MSR_IA32_XFD, xfd);
186	__this_cpu_write(xfd_state, xfd);
187}
188
189static inline void xfd_update_state(struct fpstate *fpstate)
190{
191	if (fpu_state_size_dynamic()) {
192		u64 xfd = fpstate->xfd;
193
194		if (__this_cpu_read(xfd_state) != xfd)
195			xfd_set_state(xfd);
196	}
197}
198
199extern int __xfd_enable_feature(u64 which, struct fpu_guest *guest_fpu);
200#else
201static inline void xfd_set_state(u64 xfd) { }
202
203static inline void xfd_update_state(struct fpstate *fpstate) { }
204
205static inline int __xfd_enable_feature(u64 which, struct fpu_guest *guest_fpu) {
206	return -EPERM;
207}
208#endif
209
210/*
211 * Save processor xstate to xsave area.
212 *
213 * Uses either XSAVE or XSAVEOPT or XSAVES depending on the CPU features
214 * and command line options. The choice is permanent until the next reboot.
215 */
216static inline void os_xsave(struct fpstate *fpstate)
217{
218	u64 mask = fpstate->xfeatures;
219	u32 lmask = mask;
220	u32 hmask = mask >> 32;
221	int err;
222
223	WARN_ON_FPU(!alternatives_patched);
224	xfd_validate_state(fpstate, mask, false);
225
226	XSTATE_XSAVE(&fpstate->regs.xsave, lmask, hmask, err);
227
228	/* We should never fault when copying to a kernel buffer: */
229	WARN_ON_FPU(err);
230}
231
232/*
233 * Restore processor xstate from xsave area.
234 *
235 * Uses XRSTORS when XSAVES is used, XRSTOR otherwise.
236 */
237static inline void os_xrstor(struct fpstate *fpstate, u64 mask)
238{
239	u32 lmask = mask;
240	u32 hmask = mask >> 32;
241
242	xfd_validate_state(fpstate, mask, true);
243	XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask);
244}
245
246/* Restore of supervisor state. Does not require XFD */
247static inline void os_xrstor_supervisor(struct fpstate *fpstate)
248{
249	u64 mask = xfeatures_mask_supervisor();
250	u32 lmask = mask;
251	u32 hmask = mask >> 32;
252
253	XSTATE_XRESTORE(&fpstate->regs.xsave, lmask, hmask);
254}
255
256/*
257 * XSAVE itself always writes all requested xfeatures.  Removing features
258 * from the request bitmap reduces the features which are written.
259 * Generate a mask of features which must be written to a sigframe.  The
260 * unset features can be optimized away and not written.
261 *
262 * This optimization is user-visible.  Only use for states where
263 * uninitialized sigframe contents are tolerable, like dynamic features.
264 *
265 * Users of buffers produced with this optimization must check XSTATE_BV
266 * to determine which features have been optimized out.
267 */
268static inline u64 xfeatures_need_sigframe_write(void)
269{
270	u64 xfeaures_to_write;
271
272	/* In-use features must be written: */
273	xfeaures_to_write = xfeatures_in_use();
274
275	/* Also write all non-optimizable sigframe features: */
276	xfeaures_to_write |= XFEATURE_MASK_USER_SUPPORTED &
277			     ~XFEATURE_MASK_SIGFRAME_INITOPT;
278
279	return xfeaures_to_write;
280}
281
282/*
283 * Save xstate to user space xsave area.
284 *
285 * We don't use modified optimization because xrstor/xrstors might track
286 * a different application.
287 *
288 * We don't use compacted format xsave area for backward compatibility for
289 * old applications which don't understand the compacted format of the
290 * xsave area.
291 *
292 * The caller has to zero buf::header before calling this because XSAVE*
293 * does not touch the reserved fields in the header.
294 */
295static inline int xsave_to_user_sigframe(struct xregs_state __user *buf, u32 pkru)
296{
297	/*
298	 * Include the features which are not xsaved/rstored by the kernel
299	 * internally, e.g. PKRU. That's user space ABI and also required
300	 * to allow the signal handler to modify PKRU.
301	 */
302	struct fpstate *fpstate = x86_task_fpu(current)->fpstate;
303	u64 mask = fpstate->user_xfeatures;
304	u32 lmask;
305	u32 hmask;
306	int err;
307
308	/* Optimize away writing unnecessary xfeatures: */
309	if (fpu_state_size_dynamic())
310		mask &= xfeatures_need_sigframe_write();
311
312	lmask = mask;
313	hmask = mask >> 32;
314	xfd_validate_state(fpstate, mask, false);
315
316	stac();
317	XSTATE_OP(XSAVE, buf, lmask, hmask, err);
318	clac();
319
320	if (!err)
321		err = update_pkru_in_sigframe(buf, pkru);
322
323	return err;
324}
325
326/*
327 * Restore xstate from user space xsave area.
328 */
329static inline int xrstor_from_user_sigframe(struct xregs_state __user *buf, u64 mask)
330{
331	struct xregs_state *xstate = ((__force struct xregs_state *)buf);
332	u32 lmask = mask;
333	u32 hmask = mask >> 32;
334	int err;
335
336	xfd_validate_state(x86_task_fpu(current)->fpstate, mask, true);
337
338	stac();
339	XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
340	clac();
341
342	return err;
343}
344
345/*
346 * Restore xstate from kernel space xsave area, return an error code instead of
347 * an exception.
348 */
349static inline int os_xrstor_safe(struct fpstate *fpstate, u64 mask)
350{
351	struct xregs_state *xstate = &fpstate->regs.xsave;
352	u32 lmask = mask;
353	u32 hmask = mask >> 32;
354	int err;
355
356	/* Ensure that XFD is up to date */
357	xfd_update_state(fpstate);
358
359	if (cpu_feature_enabled(X86_FEATURE_XSAVES))
360		XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
361	else
362		XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
363
364	return err;
365}
366
367
368#endif